diff --git a/litellm/integrations/arize/_utils.py b/litellm/integrations/arize/_utils.py index f719dacd148..fd912839c95 100644 --- a/litellm/integrations/arize/_utils.py +++ b/litellm/integrations/arize/_utils.py @@ -14,6 +14,7 @@ from litellm.litellm_core_utils.redact_messages import ( should_redact_message_logging, ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls from litellm.types.utils import CallTypes, StandardLoggingMCPToolCall, StandardLoggingPayload if TYPE_CHECKING: @@ -663,12 +664,11 @@ def _to_plain_dict(value): def _get_tool_calls(message) -> list | None: - """Return ``message.tool_calls`` only when it's a non-empty list. - - Works for dicts and Pydantic message objects via ``_safe_get``. + """Return the message's tool calls, Claude's server tool calls first, or + ``None`` when there are none. Works for dicts and Pydantic message objects. """ - tool_calls: Final = _safe_get(message, "tool_calls") - return tool_calls if isinstance(tool_calls, list) and tool_calls else None + tool_calls: Final = tool_calls_with_server_tool_calls(message) + return list(tool_calls) if tool_calls else None class _NormalizedToolCallFunction(TypedDict): diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index a72023eaedf..79788d35854 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -44,6 +44,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.redact_messages import should_redact_message_logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads +from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -315,7 +316,10 @@ def _to_dd_tool_calls(message: Mapping[str, object]) -> tuple[ToolCall, ...]: tool_id=_text_field(tool_call, "id"), type=_text_field(tool_call, "type", "function"), ) - for tool_call in (raw_tool_calls if isinstance(raw_tool_calls, list) else ()) + for tool_call in ( + *anthropic_server_tool_call_params(message), + *(raw_tool_calls if isinstance(raw_tool_calls, list) else ()), + ) if isinstance(tool_call, dict) for function in [_mapping_field(tool_call, "function")] ) diff --git a/litellm/integrations/helicone.py b/litellm/integrations/helicone.py index 63acc562d98..a0052069271 100644 --- a/litellm/integrations/helicone.py +++ b/litellm/integrations/helicone.py @@ -10,6 +10,23 @@ from litellm.integrations.helicone_mock_client import ( create_mock_helicone_client, should_use_helicone_mock, ) +from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls +from litellm.types.utils import ChatCompletionMessageCustomToolCall, ChatCompletionMessageToolCall + + +def _tool_use_block( + tool_call: ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall, +) -> dict[str, object]: + match tool_call: + case ChatCompletionMessageCustomToolCall(custom=custom): + return {"type": "tool_use", "id": tool_call.id, "name": custom.name, "input": custom.input} + case _: + return { + "type": "tool_use", + "id": tool_call.id, + "name": tool_call.function.name, + "input": tool_call.function.arguments, + } class HeliconeLogger: @@ -57,23 +74,8 @@ class HeliconeLogger: message = choice["message"] content = [] - if "tool_calls" in message and message["tool_calls"]: - for tool_call in message["tool_calls"]: - function = tool_call.get("function") - custom = tool_call.get("custom") - if not function and not custom: - continue - name, tool_input = ( - (function["name"], function["arguments"]) if function else (custom["name"], custom["input"]) - ) - content.append( - { - "type": "tool_use", - "id": tool_call["id"], - "name": name, - "input": tool_input, - } - ) + if tool_calls := tool_calls_with_server_tool_calls(message): + content = [_tool_use_block(tool_call) for tool_call in tool_calls] elif "content" in message and message["content"]: content = [{"type": "text", "text": message["content"]}] diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index ee75e6501e8..0117348da66 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -26,6 +26,7 @@ from litellm.integrations.opentelemetry import ( from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads +from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls from litellm.secret_managers.main import str_to_bool from litellm.types.integrations.langfuse_otel import ( LangfuseSpanAttributes, @@ -605,8 +606,8 @@ def _extract_choices_output(response_obj: _Gettable) -> str | None: message: Final = _as_gettable(first_choice.get("message", {})) if message is None: return None - tool_calls: Final = message.get("tool_calls") - if isinstance(tool_calls, list) and tool_calls: + tool_calls: Final = tool_calls_with_server_tool_calls(message) + if tool_calls: transformed_tool_calls: Final = [ entry for tool_call in tool_calls if (entry := _transformed_tool_call(response_obj, tool_call)) is not None ] diff --git a/litellm/integrations/lunary.py b/litellm/integrations/lunary.py index 6c020c78c32..08acb2c80dc 100644 --- a/litellm/integrations/lunary.py +++ b/litellm/integrations/lunary.py @@ -6,6 +6,7 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Final import packaging +from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls if TYPE_CHECKING: from openai.types.chat.chat_completion_message_custom_tool_call import Custom @@ -67,8 +68,8 @@ def parse_messages(input): } # Only add tool_calls and function_call to res if they are set - if message.get("tool_calls"): - serialized["tool_calls"] = parse_tool_calls(message.get("tool_calls")) + if tool_calls := tool_calls_with_server_tool_calls(message): + serialized["tool_calls"] = parse_tool_calls(tool_calls) return serialized diff --git a/litellm/integrations/newrelic/newrelic.py b/litellm/integrations/newrelic/newrelic.py index 3623335f599..efc796825e4 100644 --- a/litellm/integrations/newrelic/newrelic.py +++ b/litellm/integrations/newrelic/newrelic.py @@ -56,6 +56,7 @@ from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import has_attributes from litellm.litellm_core_utils.redact_messages import should_redact_message_logging +from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus from litellm.types.integrations.newrelic import NewRelicInitParams from litellm.types.utils import Message, ModelResponse, StandardLoggingPayload @@ -479,11 +480,11 @@ class NewRelicLogger(CustomLogger): content: Final = message.get("content") # Handle tool calls - if message.get("tool_calls"): + if tool_calls := tool_calls_with_server_tool_calls(message): try: - return json.dumps(message["tool_calls"]) + return json.dumps([tool_call.model_dump() for tool_call in tool_calls]) except Exception: - return str(message["tool_calls"]) + return str(tool_calls) # Handle None or missing content if content is None: diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index a9bd93e6420..0dae89b9aed 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -42,6 +42,7 @@ from litellm.litellm_core_utils.service_tier_utils import ( get_requested_service_tier, get_served_service_tier, ) +from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls from litellm.secret_managers.main import get_secret_bool, str_to_bool from litellm.types.integrations.otel_span_attributes import SpanAttributes from litellm.types.services import ServiceLoggerPayload @@ -2736,9 +2737,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): for idx, choice in enumerate(response_obj.get("choices")): if choice.get("finish_reason"): message = choice.get("message") - tool_calls = message.get("tool_calls") - if tool_calls: - kv_pairs = OpenTelemetry._tool_calls_kv_pair(tool_calls) + if tool_calls := tool_calls_with_server_tool_calls(message): + kv_pairs = OpenTelemetry._tool_calls_kv_pair(list(tool_calls)) for key, value in kv_pairs.items(): self.safe_set_attribute( span=span, diff --git a/litellm/integrations/otel/mappers/utils.py b/litellm/integrations/otel/mappers/utils.py index 80ef2dd553a..bfd50fea5d5 100644 --- a/litellm/integrations/otel/mappers/utils.py +++ b/litellm/integrations/otel/mappers/utils.py @@ -12,6 +12,8 @@ from typing import Final from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue from litellm.integrations.otel.model.payloads import LLMCallSpanData, ToolDefinition +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of +from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params @dataclass(frozen=True, slots=True) @@ -141,12 +143,13 @@ def output_messages(data: LLMCallSpanData) -> list: def message_tool_calls(message: object) -> tuple[MessageToolCall, ...]: + tool_call_values: Final = ( + *anthropic_server_tool_call_params(message), + *parts_of(message_field(message, "tool_calls")), + ) if not isinstance(message, dict): return () - tool_calls: Final = message.get("tool_calls") - if not isinstance(tool_calls, (list, tuple)) or not tool_calls: - return () - return tuple(tool_call for value in tool_calls if (tool_call := _message_tool_call(value)) is not None) + return tuple(tool_call for value in tool_call_values if (tool_call := _message_tool_call(value)) is not None) def _message_tool_call(value: object) -> MessageToolCall | None: diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index 7a62ed6aed9..4238d72fc5e 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -10,6 +10,8 @@ from types import MappingProxyType from typing import TYPE_CHECKING, ClassVar, Final, Literal, Protocol, cast from urllib.parse import urlsplit +from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch +from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm.integrations.otel.model.metadata import ( @@ -36,6 +38,7 @@ from litellm.integrations.otel.model.utils import ( as_str_tuple, ) from litellm.integrations.otel.routing import RoutingAttributeValue, routing_decision_attributes +from litellm.types.llms.anthropic import claude_hosted_web_tool_call # ``RequestIdentity`` and the request-metadata translation now live in # :mod:`metadata`; re-exported here so existing ``model.payloads`` imports keep @@ -809,6 +812,7 @@ class _Choice(TypedDict): _RESPONSES_TOOL_CALL_TYPES: Final = frozenset({"function_call", "custom_tool_call"}) +_WEB_SEARCH_CALL: Final = TypeAdapter(ResponseFunctionWebSearch) def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: @@ -816,9 +820,13 @@ def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: items: Final = _dicts(response.get("output")) messages: Final = tuple(item for item in items if item.get("type") == "message") parts: Final = tuple(part for item in messages for part in _dicts(item.get("content"))) - tool_calls: Final = tuple( + client_tool_calls: Final = tuple( _responses_tool_call(item) for item in items if item.get("type") in _RESPONSES_TOOL_CALL_TYPES ) + tool_calls: Final = ( + *(call for item in items if (call := _claude_web_search_tool_call(item)) is not None), + *client_tool_calls, + ) if not messages and not tool_calls: return () message: Final[_AssistantMessage] = { @@ -827,7 +835,10 @@ def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: "refusal": _responses_parts_text(parts, "refusal", "refusal"), "tool_calls": tool_calls or None, } - choice: Final[_Choice] = {"message": message, "finish_reason": _responses_finish_reason(response, bool(tool_calls))} + choice: Final[_Choice] = { + "message": message, + "finish_reason": _responses_finish_reason(response, bool(client_tool_calls)), + } return (choice,) @@ -981,6 +992,20 @@ def _responses_tool_call(item: Mapping[str, object]) -> _ToolCall: return tool_call +def _claude_web_search_tool_call(item: Mapping[str, object]) -> _ToolCall | None: + if item.get("type") != "web_search_call": + return None + try: + search: Final = _WEB_SEARCH_CALL.validate_python(item) + except ValidationError: + return None + claude_call: Final = claude_hosted_web_tool_call(search) + if claude_call is None: + return None + tool_id, name, arguments = claude_call + return {"id": tool_id, "type": "function", "function": {"name": name, "arguments": arguments}} + + def _responses_finish_reason(response: Mapping[str, object], has_tool_calls: bool) -> str | None: status: Final = as_str(response.get("status")) if status == "completed": diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index b208ab76ead..b4aea1c6ba5 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -39,6 +39,7 @@ from litellm.litellm_core_utils.llm_judge import ( parse_json_verdict, ) from litellm.litellm_core_utils.redact_messages import should_redact_message_logging +from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalDirection @@ -202,7 +203,7 @@ def _chat_final_text(response_obj: object) -> str: if read is None: return "" prose: Final = extract_text_from_content(read("content")) - if not (read("tool_calls") or read("function_call")): + if not _tool_calls_list(read): return prose serialized: Final = _serialize_tool_calls(read) return f"{prose} {serialized}".strip() if prose else serialized @@ -218,8 +219,9 @@ _RESPONSES_TOOL_CALL_TYPES: Final = frozenset(("function_call", "custom_tool_cal def _tool_calls_list(read: Callable[[str], object]) -> tuple[object, ...]: - calls: Final = read("tool_calls") - listed: Final = tuple(calls) if isinstance(calls, Sequence) and not isinstance(calls, str) else () + listed: Final = tool_calls_with_server_tool_calls( + {"tool_calls": read("tool_calls"), "provider_specific_fields": read("provider_specific_fields")} + ) single: Final = read("function_call") return listed if listed else ((single,) if single is not None else ()) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 862e61cb631..9500a78e9fa 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -606,6 +606,8 @@ def get_content_from_model_response(response: ModelResponse | dict) -> str: else: new_response = response + from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls + content = "" for choice in new_response.choices: @@ -613,9 +615,8 @@ def get_content_from_model_response(response: ModelResponse | dict) -> str: content += choice.message.content if choice.message.content else "" if choice.message.function_call: content += choice.message.function_call.model_dump_json() - if choice.message.tool_calls: - for tc in choice.message.tool_calls: - content += tc.model_dump_json() + for tc in tool_calls_with_server_tool_calls(choice.message): + content += tc.model_dump_json() elif isinstance(choice, StreamingChoices): content += getattr(choice, "delta", {}).get("content", "") or "" return content diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index de896cde4dc..0052cbc0c34 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1995,12 +1995,39 @@ def _tool_call_ids_with_a_kept_server_result(message: object) -> frozenset[str]: return frozenset() return frozenset( tool_call_id - for tool_call in parts_of(message_field(message, "tool_calls")) + for tool_call in ( + *parts_of(message_field(message, "tool_calls")), + *parts_of(message_field(provider_specific_fields, "server_tool_calls")), + ) if isinstance(tool_call_id := message_field(tool_call, "id"), str) and tool_call_is_rebuilt_as_server_tool_use(tool_call_id, provider_specific_fields) ) +def _assistant_tool_calls_with_server_tool_calls( + message: ChatCompletionAssistantMessage, +) -> Sequence[ChatCompletionAssistantToolCall] | None: + from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params + + tool_calls: Final = message.get("tool_calls") + server_tool_calls: Final = anthropic_server_tool_call_params(message) + if not server_tool_calls: + return tool_calls + client_tool_call_ids: Final = frozenset(tool_call["id"] for tool_call in tool_calls or ()) + return ( + *(tool_call for tool_call in server_tool_calls if tool_call["id"] not in client_tool_call_ids), + *(tool_calls or ()), + ) + + +def _server_tool_call_ids(message: object) -> frozenset[str]: + from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params + + return frozenset( + tool_call["id"] for tool_call in anthropic_server_tool_call_params(message) if tool_call["id"] is not None + ) + + def _server_tool_answers_replaced_after(messages: Sequence[object], index: int) -> frozenset[str]: kept_server_result_ids: Final = _tool_call_ids_with_a_kept_server_result(messages[index]) if not kept_server_result_ids: @@ -2060,10 +2087,11 @@ _AnthropicToolInvokeItem: TypeAlias = AnthropicMessagesToolUseParam | dict[str, def convert_to_anthropic_tool_invoke( - tool_calls: list[ChatCompletionAssistantToolCall], + tool_calls: Sequence[ChatCompletionAssistantToolCall], web_search_results: Sequence[object] | None = None, tool_results: Sequence[object] | None = None, answered_tool_ids: Container[str] = frozenset(), + server_tool_call_ids: Container[str] = frozenset(), ) -> list[_AnthropicToolInvokeItem]: """ OpenAI tool invokes: @@ -2141,6 +2169,10 @@ def convert_to_anthropic_tool_invoke( anthropic_tool_invoke.append(server_tool_result) elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids: continue + elif tool_id in server_tool_call_ids and tool_id not in answered_tool_ids: + anthropic_tool_invoke.append( + {"type": "server_tool_use", "id": tool_id, "name": tool_name, "input": tool_input} + ) else: sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id) _anthropic_tool_use_param = AnthropicMessagesToolUseParam( @@ -3100,7 +3132,7 @@ def anthropic_messages_pt( # If so, we need to interleave thinking blocks with tool call groups # to preserve the original content block ordering. # Fixes: https://github.com/BerriAI/litellm/issues/23047 - assistant_tool_calls = assistant_content_block.get("tool_calls") + assistant_tool_calls = _assistant_tool_calls_with_server_tool_calls(assistant_content_block) _has_server_tool_calls = False if assistant_tool_calls is not None: for _tc in assistant_tool_calls: @@ -3133,6 +3165,7 @@ def anthropic_messages_pt( web_search_results=_web_search_results_tc, tool_results=_tool_results_tc, answered_tool_ids=client_answered_tool_ids, + server_tool_call_ids=_server_tool_call_ids(assistant_content_block), ) # Group tool invoke results into (server_tool_use, result) pairs @@ -3298,6 +3331,7 @@ def anthropic_messages_pt( web_search_results=_web_search_results, tool_results=_tool_results, answered_tool_ids=client_answered_tool_ids, + server_tool_call_ids=_server_tool_call_ids(assistant_content_block), ) # Prevent "tool_use ids must be unique" errors by filtering duplicates diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 981859c620b..d3b420cb2e1 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -14,7 +14,7 @@ from collections.abc import Mapping, Sequence from dataclasses import replace from typing import TYPE_CHECKING, Any, Final -from pydantic import JsonValue, TypeAdapter, ValidationError +from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError import litellm from litellm.constants import REDACTED_BY_LITELLM @@ -254,9 +254,63 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[ ) if key in fields }, + **( + {"server_tool_calls": _redacted_tool_call_dicts(fields["server_tool_calls"])} + if "server_tool_calls" in fields + else {} + ), + **( + {"web_search_calls": _redacted_web_search_calls(fields["web_search_calls"])} + if "web_search_calls" in fields + else {} + ), } +def _redacted_tool_call_dict(call: object) -> object: + if (mapping := _as_mapping(call)) is None or (function := _as_mapping(mapping.get("function"))) is None: + return call + return {**mapping, "function": {**function, "arguments": REDACTED_BY_LITELLM}} + + +def _redacted_tool_call_dicts(calls: object) -> object: + return [_redacted_tool_call_dict(call) for call in calls] if isinstance(calls, list) else calls + + +_WEB_ACTION_TEXT_FIELDS: Final = ("query", "url", "pattern") +_WEB_ACTION_LIST_FIELDS: Final = ("queries", "sources") + + +def _web_action_redactions(action: Mapping[str, object]) -> Mapping[str, object]: + return { + **{key: REDACTED_BY_LITELLM for key in _WEB_ACTION_TEXT_FIELDS if action.get(key) is not None}, + **{key: None for key in _WEB_ACTION_LIST_FIELDS if action.get(key) is not None}, + } + + +def _redacted_web_search_action(action: object) -> object: + if isinstance(action, BaseModel): + return action.model_copy(update=dict(_web_action_redactions(action.model_dump()))) + mapping: Final = _as_mapping(action) + return action if mapping is None else {**mapping, **_web_action_redactions(mapping)} + + +def _redacted_web_search_call(item: object) -> object: + if isinstance(item, BaseModel): + action: Final = getattr(item, "action", None) + return item if action is None else item.model_copy(update={"action": _redacted_web_search_action(action)}) + mapping: Final = _as_mapping(item) + if mapping is None or mapping.get("action") is None: + return item + return {**mapping, "action": _redacted_web_search_action(mapping["action"])} + + +def _redacted_web_search_calls(calls: object) -> object: + if (mapping := _as_mapping(calls)) is not None: + return {key: _redacted_web_search_call(call) for key, call in mapping.items()} + return [_redacted_web_search_call(call) for call in calls] if isinstance(calls, list) else calls + + def _redact_field_when_present(item: object, field: str) -> None: if getattr(item, field, None) is not None: setattr(item, field, REDACTED_BY_LITELLM) @@ -283,6 +337,8 @@ def _redact_responses_api_output(output_items): output_item.arguments = REDACTED_BY_LITELLM if output_type == "custom_tool_call" and hasattr(output_item, "input"): output_item.input = REDACTED_BY_LITELLM + if output_type == "web_search_call" and getattr(output_item, "action", None) is not None: + output_item.action = _redacted_web_search_action(output_item.action) _RESPONSE_ITEMS: Final = TypeAdapter(tuple[object, ...]) @@ -337,6 +393,11 @@ def _redacted_responses_api_output_item(item: object, redacted_str: str) -> obje if item_mapping.get("type") == "custom_tool_call" and "input" in item_mapping else {} ), + **( + {"action": _redacted_web_search_action(item_mapping["action"])} + if item_mapping.get("type") == "web_search_call" and item_mapping.get("action") is not None + else {} + ), } diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 7f688ae733b..4188c675dfd 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -16,7 +16,7 @@ import json from collections.abc import Iterable, Mapping, MutableSequence, Sequence from copy import deepcopy from dataclasses import dataclass -from itertools import chain, repeat +from itertools import chain, filterfalse, repeat from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast, overload, runtime_checkable @@ -26,6 +26,7 @@ from typing_extensions import ReadOnly, TypedDict, assert_never from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.anthropic.common_utils import anthropic_server_tool_calls from litellm.llms.anthropic.messages.adapters.transformation import ( OPENAI_MAX_TOOL_NAME_LENGTH, LiteLLMAnthropicMessagesAdapter, @@ -205,17 +206,15 @@ class _WebSearchString: value: str -def _streamed_web_search_result_blocks(response: ModelResponse) -> tuple[Mapping[str, object], ...]: - choice: Final = response.choices[0] if response.choices else None - fields: Final = as_str_mapping(choice.message.provider_specific_fields) if isinstance(choice, Choices) else None - blocks: Final = ( - (as_str_mapping(item) for item in iter_items(fields.get("web_search_results"))) - if fields is not None and isinstance(fields.get("web_search_results"), list) - else () - ) - return tuple( - deepcopy(block) for block in blocks if block is not None and block.get("type") == "web_search_tool_result" - ) +def _legacy_hook_block(block: object) -> object: + if not isinstance(block, Mapping): + return block + fields: Final = _as_str_mapping(block) + return {**fields, "type": "tool_use"} if fields.get("type") == "server_tool_use" else fields + + +def _is_web_search_result_block(block: object) -> bool: + return isinstance(block, Mapping) and _as_str_mapping(block).get("type") == "web_search_tool_result" def _web_search_results(block: Mapping[str, object]) -> Iterable[object]: @@ -585,11 +584,14 @@ class AnthropicMessagesHandler(BaseTranslation): if not isinstance(response, ModelResponse): return response translated: Final = self.adapter.translate_openai_response_to_anthropic(response) - result_blocks: Final = _streamed_web_search_result_blocks(response) - if not result_blocks: - return translated - content: Final = translated.get("content") or () - return {**translated, "content": [*result_blocks, *content]} + blocks: Final = tuple(map(_legacy_hook_block, translated.get("content") or ())) + return { + **translated, + "content": [ + *filter(_is_web_search_result_block, blocks), + *filterfalse(_is_web_search_result_block, blocks), + ], + } @staticmethod def _build_streaming_usage_response( @@ -1543,9 +1545,9 @@ class AnthropicMessagesHandler(BaseTranslation): if built_response is not None and hasattr(built_response, "choices") and built_response.choices: model_response: Final = cast(ModelResponse, built_response) first_choice: Final = cast(Choices, model_response.choices[0]) - tool_calls_list: Final = cast( - list[ChatCompletionMessageToolCall] | None, - first_choice.message.tool_calls, + tool_calls_list: Final = ( + *anthropic_server_tool_calls(first_choice.message), + *cast(list[ChatCompletionMessageToolCall], first_choice.message.tool_calls or []), ) string_so_far = first_choice.message.content pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ()) @@ -1559,7 +1561,7 @@ class AnthropicMessagesHandler(BaseTranslation): if texts_to_check: guardrail_inputs["texts"] = texts_to_check if tool_calls_list: - guardrail_inputs["tool_calls"] = tool_calls_list + guardrail_inputs["tool_calls"] = list(tool_calls_list) try: prepared_request_data = self._prepare_request_data( diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index e1436473426..e414e333684 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -54,6 +54,7 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallFunctionChunk, ) from litellm.types.responses.main import ( + HOSTED_WEB_TOOL_NAMES, OutputCodeInterpreterCall, build_code_interpreter_log_outputs, build_web_search_call, @@ -754,6 +755,8 @@ class ModelResponseIterator: self._server_tool_inputs: dict[str, object] = {} self.tool_results: list[dict[str, Any]] = [] self._current_server_tool_id: str | None = None + self._current_server_tool_name: str = "" + self._server_tool_calls: tuple[ChatCompletionToolCallChunk, ...] = () self._container_id: str | None = None @property @@ -824,10 +827,9 @@ class ModelResponseIterator: return text, tool_use, thinking_blocks, provider_specific_fields, reasoning_content self.content_blocks.append(content_block) if "partial_json" in content_block["delta"]: - # Only emit tool calls if we're in a tool_use or server_tool_use block # web_search_tool_result blocks also have input_json_delta but should not be treated as tool calls # See: https://github.com/BerriAI/litellm/issues/17254 - if self.current_content_block_type in ("tool_use", "server_tool_use"): + if self.current_content_block_type == "tool_use": tool_use = cast( ChatCompletionToolCallChunk, { @@ -926,7 +928,7 @@ class ModelResponseIterator: def _web_search_call_snapshot(self) -> dict[str, object]: return dict(self._web_search_calls) - def _complete_web_search_call(self, result: dict[str, object]) -> None: + def _complete_web_search_call(self, result: dict[str, object], tool_name: str) -> None: tool_use_id: Final = result.get("tool_use_id") if not isinstance(tool_use_id, str) or tool_use_id not in self._web_search_calls: return @@ -934,6 +936,7 @@ class ModelResponseIterator: tool_id=tool_use_id, tool_input=self._server_tool_inputs.get(tool_use_id, {}), result=result, + tool_name=tool_name, ) def _build_code_interpreter_results(self) -> list: @@ -1013,7 +1016,6 @@ class ModelResponseIterator: content_block_start["content_block"]["type"] == "tool_use" or content_block_start["content_block"]["type"] == "server_tool_use" ): - self.tool_index += 1 # Reverse-map the (sanitized) tool name back to the # caller's original. No-op when the map is empty. _stream_tool_name = content_block_start["content_block"]["name"] @@ -1022,35 +1024,39 @@ class ModelResponseIterator: # Use empty string for arguments in content_block_start - actual arguments # come in subsequent content_block_delta chunks and get accumulated. # Using str(input) here would prepend '{}' causing invalid JSON accumulation. - tool_use = ChatCompletionToolCallChunk( - id=content_block_start["content_block"]["id"], - type="function", - function=ChatCompletionToolCallFunctionChunk( - name=_stream_tool_name, - arguments="", - ), - index=self.tool_index, - ) + if content_block_start["content_block"]["type"] == "tool_use": + self.tool_index += 1 + tool_use = ChatCompletionToolCallChunk( + id=content_block_start["content_block"]["id"], + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=_stream_tool_name, + arguments="", + ), + index=self.tool_index, + ) # Track server tool use inputs for code_interpreter_results. # The initial input in content_block_start is typically {} # for streaming; the full input arrives via input_json_delta # and is assembled at content_block_stop. if content_block_start["content_block"]["type"] == "server_tool_use": self._current_server_tool_id = content_block_start["content_block"]["id"] + self._current_server_tool_name = _stream_tool_name tool_input: Final = content_block_start["content_block"].get("input", {}) self._server_tool_inputs[self._current_server_tool_id] = tool_input - if _stream_tool_name == "web_search": + if _stream_tool_name in HOSTED_WEB_TOOL_NAMES: self._web_search_calls[self._current_server_tool_id] = build_web_search_call( self._current_server_tool_id, tool_input, {"content": []}, status="in_progress", + tool_name=_stream_tool_name, ) provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot() # Include caller information if present (for programmatic tool calling) if "caller" in content_block_start["content_block"]: caller_data: Final = content_block_start["content_block"]["caller"] - if caller_data: + if caller_data and tool_use is not None: tool_use["caller"] = cast(dict[str, Any], caller_data) elif content_block_start["content_block"]["type"] == "redacted_thinking": ( @@ -1081,7 +1087,7 @@ class ModelResponseIterator: # The full content comes in content_block_start, not in deltas # See: https://github.com/BerriAI/litellm/issues/17737 self.web_search_results.append(content_block_start["content_block"]) - self._complete_web_search_call(content_block_start["content_block"]) + self._complete_web_search_call(content_block_start["content_block"], "web_search") provider_specific_fields["web_search_results"] = self.web_search_results provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot() elif content_type == "web_fetch_tool_result": @@ -1089,7 +1095,9 @@ class ModelResponseIterator: # The full content comes in content_block_start, not in deltas # Fixes: https://github.com/BerriAI/litellm/issues/18137 self.web_search_results.append(content_block_start["content_block"]) + self._complete_web_search_call(content_block_start["content_block"], "web_fetch") provider_specific_fields["web_search_results"] = self.web_search_results + provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot() else: self.tool_results.append(content_block_start["content_block"]) provider_specific_fields["tool_results"] = self.tool_results @@ -1104,7 +1112,7 @@ class ModelResponseIterator: # check if tool call content block - only for tool_use and server_tool_use blocks if self.current_content_block_type in ("tool_use", "server_tool_use"): is_empty: Final = self.check_empty_tool_call_args() - if is_empty: + if is_empty and self.current_content_block_type == "tool_use": tool_use = ChatCompletionToolCallChunk( id=None, type="function", @@ -1128,6 +1136,29 @@ class ModelResponseIterator: self._server_tool_inputs[self._current_server_tool_id] = json.loads(args) except (json.JSONDecodeError, TypeError): pass + server_tool_input: Final = self._server_tool_inputs.get(self._current_server_tool_id, {}) + if self._current_server_tool_id in self._web_search_calls: + self._web_search_calls[self._current_server_tool_id] = build_web_search_call( + self._current_server_tool_id, + server_tool_input, + {"content": []}, + status="in_progress", + tool_name=self._current_server_tool_name, + ) + provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot() + self._server_tool_calls = ( + *self._server_tool_calls, + ChatCompletionToolCallChunk( + id=self._current_server_tool_id, + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=self._current_server_tool_name, + arguments=json.dumps(server_tool_input), + ), + index=len(self._server_tool_calls), + ), + ) + provider_specific_fields["server_tool_calls"] = list(self._server_tool_calls) self._current_server_tool_id = None if self.is_response_format_tool: text = self._flush_response_format_buffer() diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index d773e31c733..5660747638e 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -7,6 +7,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, cast import httpx +from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict @@ -90,11 +91,11 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ChatCompletionToolParam, - OpenAIChatCompletionFinishReason, OpenAIMcpServerTool, OpenAIWebSearchOptions, ) from litellm.types.responses.main import ( + HOSTED_WEB_TOOL_NAMES, OutputCodeInterpreterCall, build_code_interpreter_log_outputs, build_web_search_call, @@ -166,6 +167,7 @@ _ANTHROPIC_TOOL_NAME_MAX_LEN: Final = 128 _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool) _TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _TOOLS_ADAPTER: Final = TypeAdapter(list[object]) +_CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(tuple[object, ...]) _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset( ("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid") @@ -252,6 +254,42 @@ def _validated_defer_loading(value: object) -> bool: raise ValueError("defer_loading must be a boolean") from exc +def _hosted_web_tool_use(block: object) -> tuple[str, str, object] | None: + match block: + case { + "type": "server_tool_use", + "name": str() as name, + "id": str() as call_id, + "input": object() as tool_input, + } if name in HOSTED_WEB_TOOL_NAMES: + return call_id, name, tool_input + case _: + return None + + +def _server_tool_use_id(block: object) -> str | None: + match block: + case {"type": "server_tool_use", "id": str() as call_id}: + return call_id + case _: + return None + + +def _server_tool_use_ids(completion_response: Mapping[str, object]) -> frozenset[str]: + blocks: Final = _CONTENT_BLOCKS_ADAPTER.validate_python(completion_response.get("content") or ()) + return frozenset(filter(None, map(_server_tool_use_id, blocks))) + + +def _hosted_web_call(call_id: str, tool_name: str, tool_input: object, result: object) -> ResponseFunctionWebSearch: + return build_web_search_call( + tool_id=call_id, + tool_input=tool_input, + result=result, + status=None if result is not None else "in_progress", + tool_name=tool_name, + ) + + class _AnthropicUsageIteration(TypedDict, total=False): """One entry of the ``usage.iterations`` array on an Anthropic response.""" @@ -2896,29 +2934,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self, web_search_results: Sequence[object], completion_response: Mapping[str, object], - ) -> list[object]: + ) -> list[ResponseFunctionWebSearch]: content: Final = completion_response.get("content") blocks: Final = content if isinstance(content, Sequence) else () - inputs: Final = { - call_id: tool_input - for block in blocks - if isinstance(block, Mapping) - and block.get("type") == "server_tool_use" - and block.get("name") == "web_search" - and isinstance((call_id := block.get("id")), str) - and isinstance((tool_input := block.get("input")), Mapping) + results_by_call: Final = { + (tool_use_id, result.get("type")): result + for result in web_search_results + if isinstance(result, dict) and isinstance((tool_use_id := result.get("tool_use_id")), str) } return [ - build_web_search_call( - tool_id=tool_use_id, - tool_input=inputs.get(tool_use_id, {}), - result=result, - ) - for result in web_search_results - if isinstance(result, dict) - and result.get("type") == "web_search_tool_result" - and isinstance((tool_use_id := result.get("tool_use_id")), str) - and tool_use_id in inputs + _hosted_web_call(call_id, tool_name, tool_input, results_by_call.get((call_id, f"{tool_name}_tool_result"))) + for call_id, tool_name, tool_input in filter(None, map(_hosted_web_tool_use, blocks)) ] def _build_provider_specific_fields( @@ -2930,6 +2956,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): tool_results: Sequence[_AnthropicToolResultBlock] | None, compaction_blocks: Sequence[object] | None, tool_calls: list[ChatCompletionToolCallChunk], + server_tool_use_ids: frozenset[str], ) -> dict[str, object]: provider_specific_fields: Final[dict[str, object]] = { "citations": citations, @@ -2942,10 +2969,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if web_search_results is not None: provider_specific_fields["web_search_results"] = web_search_results - provider_specific_fields["web_search_calls"] = self._build_web_search_calls( - web_search_results, - completion_response, - ) + + web_search_calls: Final = self._build_web_search_calls(web_search_results or (), completion_response) + if web_search_calls: + provider_specific_fields["web_search_calls"] = web_search_calls if tool_results is not None: provider_specific_fields["tool_results"] = tool_results @@ -2967,6 +2994,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if compaction_blocks is not None: provider_specific_fields["compaction_blocks"] = compaction_blocks + server_tool_calls: Final = [tc for tc in tool_calls if tc["id"] in server_tool_use_ids] + if server_tool_calls: + provider_specific_fields["server_tool_calls"] = server_tool_calls + refusal_fields: Final = refusal_provider_specific_fields( REFUSAL_STOP_FIELDS.validate_python(completion_response) ) @@ -3026,6 +3057,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ): text_content = prefix_prompt + text_content + server_tool_use_ids: Final = _server_tool_use_ids(completion_response) + client_tool_calls: Final = [tc for tc in tool_calls if tc["id"] not in server_tool_use_ids] provider_specific_fields: Final = self._build_provider_specific_fields( completion_response, citations, @@ -3034,11 +3067,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): tool_results, compaction_blocks, tool_calls, + server_tool_use_ids, ) json_mode_message, tool_calls_for_message, json_extra_content = self._resolve_json_mode_non_streaming( json_mode=json_mode, - tool_calls=tool_calls, + tool_calls=client_tool_calls, response_format_schema=response_format_schema, ) merged_text = text_content or "" @@ -3068,10 +3102,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): model_response.choices[0].message = _message model_response.hidden_params["original_response"] = completion_response["content"] - model_response.choices[0].finish_reason = cast( - OpenAIChatCompletionFinishReason, - map_finish_reason(completion_response["stop_reason"]), - ) + model_response.choices[0].finish_reason = map_finish_reason(completion_response["stop_reason"]) usage: Final = self.calculate_usage( usage_object=completion_response["usage"], diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index e1f9a7d540f..cbd70528ca1 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -65,10 +65,26 @@ from litellm.types.llms.anthropic import ( AnthropicThinkingParam, ) from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionAssistantToolCall, + ChatCompletionToolCallFunctionChunk, +) from litellm.types.proxy.auth.special_headers import SpecialHeaders from litellm.types.proxy.model_listing import ModelInfoResponse -from litellm.types.utils import LlmProviders +from litellm.types.responses.main import ( + ResponseFunctionWebSearch, + web_search_call_tool_id, + with_hosted_web_tool_arguments, +) +from litellm.types.utils import ( + ChatCompletionMessageCustomToolCall, + ChatCompletionMessageToolCall, + Delta, + LlmProviders, + Message, + chat_completion_tool_call_from_dict, +) _MessageT = TypeVar("_MessageT") @@ -121,6 +137,11 @@ _DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)") _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:" OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) +_SERVER_TOOL_CALLS_ADAPTER: Final = TypeAdapter(tuple[ChatCompletionAssistantToolCall, ...]) +_SERVER_TOOL_CALL_ADAPTER: Final = TypeAdapter(ChatCompletionAssistantToolCall) +_SERVER_CALL_FIELD_KEYS: Final = ("server_tool_calls", "web_search_calls") +_WEB_SEARCH_CALL_ADAPTER: Final = TypeAdapter(ResponseFunctionWebSearch) +_TOOL_CALL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _DECODED_JSON: Final = TypeAdapter(object) _JSON_VALUE: Final = TypeAdapter(JsonValue) @@ -2209,6 +2230,179 @@ def _replayed_server_tool_use(block: object) -> _ReplayedServerToolUse | None: return None +def anthropic_server_tool_call_params(message: object) -> tuple[ChatCompletionAssistantToolCall, ...]: + fields: Final = validated_object_mapping(message_field(message, "provider_specific_fields")) + try: + return _SERVER_TOOL_CALLS_ADAPTER.validate_python(fields.get("server_tool_calls", ()) if fields else ()) + except ValidationError: + return () + + +def anthropic_server_tool_calls(message: object) -> tuple[ChatCompletionMessageToolCall, ...]: + return tuple(_message_tool_call_from_params(call) for call in anthropic_server_tool_call_params(message)) + + +def tool_calls_with_server_tool_calls( + message: object, +) -> tuple[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall, ...]: + client_tool_calls: Final = tuple( + call + for tool_call in parts_of(message_field(message, "tool_calls")) + if (call := _client_message_tool_call(tool_call)) is not None + ) + return (*anthropic_server_tool_calls(message), *client_tool_calls) + + +def anthropic_server_call_fields(holder: object) -> Mapping[str, object]: + fields: Final = validated_object_mapping(message_field(holder, "provider_specific_fields")) or {} + return MappingProxyType({key: fields[key] for key in _SERVER_CALL_FIELD_KEYS if fields.get(key)}) + + +def message_dict_with_server_tool_calls(message: object) -> object: + mapping: Final = validated_object_mapping(message) + server_tool_calls: Final = anthropic_server_tool_call_params(message) + if mapping is None or not server_tool_calls: + return message + return {**mapping, "tool_calls": [*server_tool_calls, *parts_of(mapping.get("tool_calls"))]} + + +def _server_tool_call_param(tool_call: object) -> ChatCompletionAssistantToolCall | None: + try: + call: Final = _SERVER_TOOL_CALL_ADAPTER.validate_python(tool_call) + except ValidationError: + return None + return call if _server_tool_call_id(call).startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) else None + + +def message_dict_with_server_tool_calls_split_out(message: object) -> object: + mapping: Final = validated_object_mapping(message) + tool_calls: Final = parts_of(mapping.get("tool_calls")) if mapping is not None else () + server_tool_calls: Final = tuple( + call for tool_call in tool_calls if (call := _server_tool_call_param(tool_call)) is not None + ) + if mapping is None or not server_tool_calls: + return message + fields: Final = validated_object_mapping(mapping.get("provider_specific_fields")) or {} + changed: Final = changed_server_tool_call_arguments(anthropic_server_tool_call_params(message), server_tool_calls) + client_tool_calls: Final = [tool_call for tool_call in tool_calls if _server_tool_call_param(tool_call) is None] + return { + **mapping, + "tool_calls": client_tool_calls or None, + "provider_specific_fields": { + **fields, + "server_tool_calls": list(server_tool_calls), + **( + {"web_search_calls": _rewritten_web_search_calls(fields["web_search_calls"], changed)} + if changed and fields.get("web_search_calls") is not None + else {} + ), + }, + } + + +def with_server_tool_call_arguments( + call: ChatCompletionAssistantToolCall, arguments: str +) -> ChatCompletionAssistantToolCall: + return ChatCompletionAssistantToolCall( + id=call["id"], + type=call["type"], + function=ChatCompletionToolCallFunctionChunk(name=call["function"].get("name"), arguments=arguments), + ) + + +def write_anthropic_server_tool_calls( + holder: Message | Delta, rewritten: tuple[ChatCompletionAssistantToolCall, ...] +) -> None: + original: Final = anthropic_server_tool_call_params(holder) + if rewritten == original: + return + holder.provider_specific_fields = { # rebind-ok: a guardrail rewrite must land on the message it scanned + **(holder.provider_specific_fields or {}), + "server_tool_calls": list(rewritten), + } + write_anthropic_hosted_web_call_arguments(holder, changed_server_tool_call_arguments(original, rewritten)) + + +def _server_tool_call_id(call: ChatCompletionAssistantToolCall) -> str: + return call["id"] or "" + + +def _server_tool_call_arguments(call: ChatCompletionAssistantToolCall) -> str: + return call["function"].get("arguments") or "" + + +def changed_server_tool_call_arguments( + original: Sequence[ChatCompletionAssistantToolCall], rewritten: Sequence[ChatCompletionAssistantToolCall] +) -> Mapping[str, str]: + original_arguments: Final = MappingProxyType( + {_server_tool_call_id(call): _server_tool_call_arguments(call) for call in original} + ) + return MappingProxyType( + { + _server_tool_call_id(call): _server_tool_call_arguments(call) + for call in rewritten + if _server_tool_call_arguments(call) != original_arguments.get(_server_tool_call_id(call)) + } + ) + + +def write_anthropic_hosted_web_call_arguments(holder: Message | Delta, arguments_by_tool_id: Mapping[str, str]) -> None: + fields: Final = holder.provider_specific_fields + if not arguments_by_tool_id or not fields or fields.get("web_search_calls") is None: + return + holder.provider_specific_fields = { # rebind-ok: the hosted copy of a rewritten call must not keep its old input + **fields, + "web_search_calls": _rewritten_web_search_calls(fields["web_search_calls"], arguments_by_tool_id), + } + + +def _rewritten_web_search_calls(calls: object, arguments_by_tool_id: Mapping[str, str]) -> object: + mapping: Final = validated_object_mapping(calls) + if mapping is not None: + return {key: _with_hosted_web_call_arguments(call, arguments_by_tool_id) for key, call in mapping.items()} + if not isinstance(calls, list): + return calls + return [_with_hosted_web_call_arguments(call, arguments_by_tool_id) for call in parts_of(calls)] + + +def _with_hosted_web_call_arguments(item: object, arguments_by_tool_id: Mapping[str, str]) -> object: + try: + call: Final = _WEB_SEARCH_CALL_ADAPTER.validate_python(item) + except ValidationError: + return item + arguments: Final = arguments_by_tool_id.get(web_search_call_tool_id(call.id) or "") + if arguments is None: + return item + return call.model_copy(update={"action": with_hosted_web_tool_arguments(call.action, arguments)}) + + +def write_anthropic_server_tool_call_arguments(holder: Message | Delta, index: int, arguments: str) -> None: + write_anthropic_server_tool_calls( + holder, + tuple( + with_server_tool_call_arguments(call, arguments) if call_index == index else call + for call_index, call in enumerate(anthropic_server_tool_call_params(holder)) + ), + ) + + +def _message_tool_call_from_params(call: ChatCompletionAssistantToolCall) -> ChatCompletionMessageToolCall: + return ChatCompletionMessageToolCall(id=call["id"], type=call["type"], function=dict(call["function"])) + + +def _client_message_tool_call( + tool_call: object, +) -> ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall | None: + match tool_call: + case ChatCompletionMessageToolCall() | ChatCompletionMessageCustomToolCall(): + return tool_call + case _: + try: + return chat_completion_tool_call_from_dict(_TOOL_CALL_FIELDS_ADAPTER.validate_python(tool_call)) + except (TypeError, ValidationError): + return None + + def tool_call_is_rebuilt_as_server_tool_use( tool_call_id: object, provider_specific_fields: object, answered_tool_ids: Container[str] = frozenset() ) -> bool: diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index bc9649ed867..98b4a90c645 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -46,6 +46,25 @@ def _thought_signature(provider_specific_fields: object) -> str | None: return signature if isinstance(signature, str) else None +def _object_sequence(value: object) -> Sequence[object] | None: + return value if isinstance(value, (list, tuple)) else None + + +def _server_tool_blocks(message: object) -> tuple[Mapping[str, object], ...]: + server_tool_calls: Final = anthropic_server_tool_call_params(message) + if not server_tool_calls: + return () + fields: Final = _as_string_mapping(_optional_attr(message, "provider_specific_fields")) or {} + return tuple( + convert_to_anthropic_tool_invoke( + server_tool_calls, + web_search_results=_object_sequence(fields.get("web_search_results")), + tool_results=_object_sequence(fields.get("tool_results")), + server_tool_call_ids=frozenset(tool_call["id"] for tool_call in server_tool_calls), + ) + ) + + def _compaction_blocks(provider_specific_fields: object) -> tuple[Mapping[str, object], ...]: fields: Final = _as_string_mapping(provider_specific_fields) raw_blocks: Final = fields.get("compaction_blocks") if fields is not None else None @@ -131,12 +150,14 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, + convert_to_anthropic_tool_invoke, ) from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking, reasoning_effort_from_thinking_silently, ) from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, eager_input_streaming_flag, is_empty_unsigned_thinking_block, normalize_anthropic_tool_use_id, @@ -1462,6 +1483,7 @@ class LiteLLMAnthropicMessagesAdapter: ).model_dump() ) + new_content.extend(dict(block) for block in _server_tool_blocks(choice.message)) # Handle text content if choice.message.content is not None and (choice.message.content != "" or not compaction_blocks): new_content.append( diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index fb886e42780..bac95918599 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -4,11 +4,13 @@ import json from collections.abc import Callable, Iterator, Mapping, Sequence from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row has no typed constructor across roles +from openai.types.responses import ResponseFunctionWebSearch from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm.llms.custom_httpx.sse_lines import split_sse_lines from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage +from litellm.types.responses.main import with_hosted_web_tool_arguments _DECODED_JSON: Final = TypeAdapter(object) @@ -135,6 +137,30 @@ def stream_item_field(item: object, field: str) -> object | None: return getattr(item, field, None) +def web_search_call(item: object) -> ResponseFunctionWebSearch | None: + if isinstance(item, ResponseFunctionWebSearch): + return item + if stream_item_field(item, "type") != "web_search_call": + return None + try: + return ResponseFunctionWebSearch.model_validate(item.model_dump() if isinstance(item, BaseModel) else item) + except ValidationError: + return None + + +def with_web_search_call_arguments(item: object, arguments: str) -> object: + search: Final = web_search_call(item) + if search is None: + return item + action: Final = with_hosted_web_tool_arguments(search.action, arguments) + if isinstance(item, ResponseFunctionWebSearch): + return item.model_copy(update={"action": action}) + if isinstance(item, BaseModel): + return item.model_copy(update={"action": action.model_dump()}) + fields: Final = cast("Mapping[str, object]", item) # cast-ok: a validated web_search_call dict is str-keyed JSON + return {**fields, "action": action.model_dump()} + + def stream_item_fingerprint(item: object) -> str: plain: Final = item.model_dump() if isinstance(item, BaseModel) else item return json.dumps(plain, sort_keys=True, default=str) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 5c5204ef2f4..51dae634ab0 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -21,6 +21,7 @@ from collections.abc import Iterator, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Union, cast +from pydantic import TypeAdapter, ValidationError from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm @@ -29,6 +30,12 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i replayed_tool_search_tool_names, ) from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + changed_server_tool_call_arguments, + write_anthropic_hosted_web_call_arguments, + write_anthropic_server_tool_calls, +) from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, StreamingScanKey, @@ -50,7 +57,12 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( stream_item_items, unappliable_request_rewrite, ) -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionAssistantToolCall, + ChatCompletionToolCallFunctionChunk, + ChatCompletionToolParam, +) from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( coerce_stream_holdback_value, ) @@ -436,9 +448,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation): text_task_mappings=text_task_mappings, tool_call_task_mappings=tool_call_task_mappings, ) + server_tool_call_tasks: Final = tuple(_server_tool_call_tasks(response.choices)) + tool_calls_with_server_calls: Final = [ + *tool_calls_to_check, + *(_server_tool_call_input(call) for _, call in server_tool_call_tasks), + ] # Step 2: Apply guardrail to all texts and tool calls in batch - if texts_to_check or tool_calls_to_check: + if texts_to_check or tool_calls_with_server_calls: # Use the real request_data if provided (proxy path), otherwise # create a standalone dict (SDK / direct-call path). if request_data is None: @@ -452,8 +469,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) if images_to_check: inputs["images"] = images_to_check - if tool_calls_to_check: - inputs["tool_calls"] = tool_calls_to_check + if tool_calls_with_server_calls: + inputs["tool_calls"] = tool_calls_with_server_calls # Include model information from the response if available if hasattr(response, "model") and response.model: inputs["model"] = response.model @@ -469,8 +486,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation): returned_tool_calls: Final = guardrailed_inputs.get("tool_calls") guardrailed_tool_calls: Final[list[dict[str, object]]] = ( cast(list[dict[str, object]], returned_tool_calls) - if isinstance(returned_tool_calls, list) and len(returned_tool_calls) == len(tool_calls_to_check) - else tool_calls_to_check + if isinstance(returned_tool_calls, list) + and len(returned_tool_calls) == len(tool_calls_with_server_calls) + else tool_calls_with_server_calls ) # Step 3: Map guardrail responses back to original response structure @@ -488,6 +506,11 @@ class OpenAIChatCompletionsHandler(BaseTranslation): tool_calls=guardrailed_tool_calls, task_mappings=tool_call_task_mappings, ) + self._apply_guardrail_responses_to_output_server_tool_calls( + response=response, + guardrailed_tool_calls=guardrailed_tool_calls[len(tool_calls_to_check) :], + task_mappings=tuple(mapping for mapping, _ in server_tool_call_tasks), + ) verbose_proxy_logger.debug("OpenAI Chat Completions: Processed output response: %s", response) @@ -681,6 +704,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): pre_guardrail_tool_calls=pre_guardrail_tool_calls, guardrail_name=guardrail_to_apply.guardrail_name or "unknown", ) + self._write_ended_stream_server_tool_call_rewrites( + responses_so_far=responses_so_far, + guardrailed_response=model_response, + ) @staticmethod def _rebuild_ended_stream_per_choice( @@ -934,6 +961,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if choice.message.tool_calls and isinstance(choice.message.tool_calls, list): if len(choice.message.tool_calls) > 0: return True + if anthropic_server_tool_call_params(choice.message): + return True elif isinstance(response, ModelResponseStream): for streaming_choice in response.choices: if isinstance(streaming_choice, litellm.StreamingChoices): @@ -944,6 +973,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if streaming_choice.delta.tool_calls and isinstance(streaming_choice.delta.tool_calls, list): if len(streaming_choice.delta.tool_calls) > 0: return True + if anthropic_server_tool_call_params(streaming_choice.delta): + return True return False def _extract_output_text_images_and_tool_calls( @@ -1094,6 +1125,22 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if "name" in func_dict: existing_tool_call.function.name = func_dict["name"] + @staticmethod + def _apply_guardrail_responses_to_output_server_tool_calls( + response: "ModelResponse", + guardrailed_tool_calls: Sequence[object], + task_mappings: Sequence[tuple[int, int]], + ) -> None: + guardrailed_by_task: Final = MappingProxyType(dict(zip(task_mappings, guardrailed_tool_calls))) + for choice_idx, choice in enumerate(response.choices): + write_anthropic_server_tool_calls( + choice.message, + tuple( + _rewritten_server_tool_call(server_tool_call, guardrailed_by_task.get((choice_idx, server_idx))) + for server_idx, server_tool_call in enumerate(anthropic_server_tool_call_params(choice.message)) + ), + ) + @staticmethod def _string_choice_contents(response: "ModelResponse") -> tuple[str | None, ...]: return tuple( @@ -1206,6 +1253,42 @@ class OpenAIChatCompletionsHandler(BaseTranslation): for fragment in tail: fragment.function.arguments = "" + @staticmethod + def _write_ended_stream_server_tool_call_rewrites( + responses_so_far: Sequence["ModelResponseStream"], + guardrailed_response: "ModelResponse", + ) -> None: + guardrailed_by_id: Final = MappingProxyType( + { + (guardrailed_response.choices[choice_position].index, server_tool_call["id"]): server_tool_call + for (choice_position, _), server_tool_call in _server_tool_call_tasks(guardrailed_response.choices) + } + ) + if not guardrailed_by_id: + return + original_by_id: Final = MappingProxyType(dict(_streamed_server_tool_calls(responses_so_far))) + changed_by_choice: Final = MappingProxyType( + { + choice_index: changed_server_tool_call_arguments( + _choice_server_tool_calls(original_by_id, choice_index), + _choice_server_tool_calls(guardrailed_by_id, choice_index), + ) + for choice_index in frozenset(index for index, _ in guardrailed_by_id) + } + ) + for response in responses_so_far: + for choice in response.choices: + write_anthropic_server_tool_calls( + choice.delta, + tuple( + guardrailed_by_id.get((choice.index, server_tool_call["id"]), server_tool_call) + for server_tool_call in anthropic_server_tool_call_params(choice.delta) + ), + ) + write_anthropic_hosted_web_call_arguments( + choice.delta, changed_by_choice.get(choice.index, MappingProxyType({})) + ) + async def _apply_guardrail_responses_to_output_streaming( self, responses: list["ModelResponseStream"], @@ -1395,7 +1478,57 @@ def _stream_chunk_choices(item: object) -> Sequence[object]: def _streamed_delta_tool_calls(delta: object) -> tuple[object, ...]: function_call: Final = stream_item_field(delta, "function_call") legacy: Final = () if function_call is None else (function_call,) - return stream_item_items(delta, "tool_calls") + legacy + return anthropic_server_tool_call_params(delta) + stream_item_items(delta, "tool_calls") + legacy + + +def _server_tool_call_tasks( + choices: Sequence[object], +) -> Iterator[tuple[tuple[int, int], ChatCompletionAssistantToolCall]]: + for choice_idx, choice in enumerate(choices): + if not isinstance(choice, Choices): + continue + for server_idx, server_tool_call in enumerate(anthropic_server_tool_call_params(choice.message)): + yield (choice_idx, server_idx), server_tool_call + + +def _streamed_server_tool_calls( + responses: Sequence["ModelResponseStream"], +) -> Iterator[tuple[tuple[int, str | None], ChatCompletionAssistantToolCall]]: + for response in responses: + for choice in response.choices: + for server_tool_call in anthropic_server_tool_call_params(choice.delta): + yield (choice.index, server_tool_call["id"]), server_tool_call + + +def _choice_server_tool_calls( + calls_by_id: Mapping[tuple[int, str | None], ChatCompletionAssistantToolCall], choice_index: int +) -> tuple[ChatCompletionAssistantToolCall, ...]: + return tuple(call for (index, _), call in calls_by_id.items() if index == choice_index) + + +def _server_tool_call_input(server_tool_call: ChatCompletionAssistantToolCall) -> dict[str, object]: + return { + "id": server_tool_call["id"], + "type": server_tool_call["type"], + "function": dict(server_tool_call["function"]), + } + + +_TOOL_CALL_FUNCTION_ADAPTER: Final = TypeAdapter(ChatCompletionToolCallFunctionChunk) + + +def _rewritten_server_tool_call( + server_tool_call: ChatCompletionAssistantToolCall, guardrailed: object +) -> ChatCompletionAssistantToolCall: + try: + function: Final = _TOOL_CALL_FUNCTION_ADAPTER.validate_python(stream_item_field(guardrailed, "function")) + except ValidationError: + return server_tool_call + return ChatCompletionAssistantToolCall( + id=server_tool_call["id"], + type=server_tool_call["type"], + function={**server_tool_call["function"], **function}, + ) def _released_choices(responses_so_far: Sequence[object]) -> Iterator[object]: diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 559daf767c2..3d30b909da8 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -66,6 +66,8 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( stream_item_fingerprint, stream_item_items, unappliable_request_rewrite, + web_search_call, + with_web_search_call_arguments, ) from litellm.llms.openai.responses.guardrail_translation.tool_merge import ( append_returned_tools, @@ -81,12 +83,14 @@ from litellm.responses.litellm_completion_transformation.tool_search import is_s from litellm.responses.litellm_completion_transformation.transformation import ( TOOL_CALL_OUTPUT_ITEM_TYPES, LiteLLMCompletionResponsesConfig, + claude_web_search_call_tool_call, tool_output_to_chat_content, ) from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ( AllMessageValues, BaseLiteLLMOpenAIResponseObject, + ChatCompletionAssistantToolCall, ChatCompletionToolCallChunk, ChatCompletionToolParam, ContentPartAddedEvent, @@ -260,6 +264,8 @@ _TERMINAL_ENVELOPE_EVENT_TYPES: Final = frozenset( _TOOL_CALL_ITEM_TYPES: Final = frozenset({"function_call", "custom_tool_call"}) _TOOL_SEARCH_CALL_TYPE: Final = "tool_search_call" +_WEB_SEARCH_CALL_TYPE: Final = "web_search_call" +_ITEM_ID_KEYED_TOOL_CALL_TYPES: Final = frozenset({_TOOL_SEARCH_CALL_TYPE, _WEB_SEARCH_CALL_TYPE}) _OUTPUT_TOOL_CALL_ITEM_TYPES: Final = _TOOL_CALL_ITEM_TYPES | {_TOOL_SEARCH_CALL_TYPE} _SCANNED_TOOL_SEARCH_NAME: Final = "tool_search" _TOOL_SEARCH_ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object]) @@ -328,8 +334,27 @@ def _is_tool_call_item(item: object) -> bool: return isinstance(item, Mapping) and item.get("type") in _TOOL_CALL_ITEM_TYPES +def _claude_web_search_tool_call(item: object) -> ChatCompletionAssistantToolCall | None: + search: Final = web_search_call(item) + return None if search is None else claude_web_search_call_tool_call(search) + + +def _claude_web_tool_call(item: object, index: int) -> ChatCompletionToolCallChunk | None: + tool_call: Final = _claude_web_search_tool_call(item) + if tool_call is None: + return None + return ChatCompletionToolCallChunk(id=tool_call["id"], type="function", function=tool_call["function"], index=index) + + +def _is_output_tool_call_item(item: object) -> bool: + return ( + stream_item_field(item, "type") in _OUTPUT_TOOL_CALL_ITEM_TYPES + or _claude_web_search_tool_call(item) is not None + ) + + def _tool_call_output_item_mapping(item: object) -> Mapping[str, object] | None: - if stream_item_field(item, "type") not in _OUTPUT_TOOL_CALL_ITEM_TYPES: + if not _is_output_tool_call_item(item): return None if isinstance(item, Mapping): return cast("Mapping[str, object]", item) # cast-ok: output items are str-keyed JSON objects @@ -341,7 +366,7 @@ def _tool_call_item_key(item: object) -> str | None: if isinstance(call_id, str) and call_id: return call_id item_id: Final = stream_item_field(item, "id") - if stream_item_field(item, "type") == _TOOL_SEARCH_CALL_TYPE and isinstance(item_id, str) and item_id: + if stream_item_field(item, "type") in _ITEM_ID_KEYED_TOOL_CALL_TYPES and isinstance(item_id, str) and item_id: return item_id return None @@ -361,6 +386,17 @@ def _scanned_tool_call_item(item: Mapping[str, object]) -> Mapping[str, object]: return {**item, "name": _SCANNED_TOOL_SEARCH_NAME, "arguments": json.dumps(item.get("arguments") or {})} +def _scanned_tool_call(item: Mapping[str, object], index: int) -> ChatCompletionToolCallChunk: + web_tool_call: Final = _claude_web_tool_call(item, index) + if web_tool_call is not None: + return web_tool_call + return tool_call_dict_from_output_item(_scanned_tool_call_item(item), index) + + +def _rewritten_web_search_call(item: object, payload: str | None) -> object: + return item if payload is None else with_web_search_call_arguments(item, payload) + + def _tool_search_rewrite(payload: str) -> Mapping[str, object]: try: return {"arguments": _TOOL_SEARCH_ARGUMENTS_ADAPTER.validate_json(payload)} @@ -1299,17 +1335,22 @@ class OpenAIResponsesHandler(BaseTranslation): # delivered, so a truncated buffer still reports text undeliverable. # # ------------------------------------------------------------------ # if final_chunk.get("type") == "response.output_item.done": - finished_summary: Final = _output_compaction_summary(stream_item_field(final_chunk, "item")) + finished_item: Final = stream_item_field(final_chunk, "item") + finished_summary: Final = _output_compaction_summary(finished_item) model_response_stream: Final = ( OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(final_chunk) ) - tool_calls: Final = model_response_stream.choices[0].delta.tool_calls + finished_web_tool_call: Final = _claude_web_tool_call(finished_item, 0) + tool_calls: Final = [ + *cast(list[ChatCompletionToolCallChunk], model_response_stream.choices[0].delta.tool_calls or []), + *(() if finished_web_tool_call is None else (finished_web_tool_call,)), + ] if tool_calls or finished_summary is not None: inputs = GenericGuardrailAPIInputs() if finished_summary is not None: inputs["texts"] = [finished_summary] if tool_calls: - inputs["tool_calls"] = cast(list[ChatCompletionToolCallChunk], tool_calls) + inputs["tool_calls"] = tool_calls if hasattr(model_response_stream, "model") and model_response_stream.model: inputs["model"] = model_response_stream.model await guardrail_to_apply.apply_guardrail( @@ -1598,7 +1639,7 @@ class OpenAIResponsesHandler(BaseTranslation): if event_type not in _OUTPUT_ITEM_EVENT_TYPES: return None item: Final = stream_item_field(event, "item") - return _tool_call_item_key(item) if stream_item_field(item, "type") in _OUTPUT_TOOL_CALL_ITEM_TYPES else None + return _tool_call_item_key(item) if _is_output_tool_call_item(item) else None @staticmethod def _write_tool_call_item(item: object, name: str | None, payload: str | None) -> object: @@ -1612,6 +1653,8 @@ class OpenAIResponsesHandler(BaseTranslation): return item if item_type == _TOOL_SEARCH_CALL_TYPE: return _rewritten_tool_search_call(item, payload) + if item_type == _WEB_SEARCH_CALL_TYPE: + return _rewritten_web_search_call(item, payload) if name is not None: OpenAIResponsesHandler._write_event_field(item, "name", name) if payload is not None and isinstance(item_type, str) and item_type in _TOOL_CALL_PAYLOAD_FIELDS: @@ -1667,7 +1710,7 @@ class OpenAIResponsesHandler(BaseTranslation): stream_item_field(event, "type") in _TOOL_CALL_PAYLOAD_EVENT_TYPES or ( stream_item_field(event, "type") in _OUTPUT_ITEM_EVENT_TYPES - and stream_item_field(stream_item_field(event, "item"), "type") in _OUTPUT_TOOL_CALL_ITEM_TYPES + and _is_output_tool_call_item(stream_item_field(event, "item")) ) for event in responses_so_far ) @@ -1675,9 +1718,7 @@ class OpenAIResponsesHandler(BaseTranslation): @staticmethod def _completed_response_scan_key(response: object) -> StreamingScanKey: output_items: Final = stream_item_items(response, "output") - message_items: Final = tuple( - item for item in output_items if stream_item_field(item, "type") not in _OUTPUT_TOOL_CALL_ITEM_TYPES - ) + message_items: Final = tuple(item for item in output_items if not _is_output_tool_call_item(item)) message_parts: Final = chain.from_iterable(stream_item_items(item, "content") for item in message_items) part_texts: Final = tuple( text for part in message_parts if isinstance(text := stream_item_field(part, "text"), str) and text @@ -1685,11 +1726,7 @@ class OpenAIResponsesHandler(BaseTranslation): summaries: Final = tuple(summary for item in output_items if (summary := _output_compaction_summary(item))) return StreamingScanKey( texts=(*part_texts, *summaries), - tool_calls=tuple( - stream_item_fingerprint(item) - for item in output_items - if stream_item_field(item, "type") in _OUTPUT_TOOL_CALL_ITEM_TYPES - ), + tool_calls=tuple(stream_item_fingerprint(item) for item in output_items if _is_output_tool_call_item(item)), stream_ended=True, ) @@ -1802,9 +1839,7 @@ class OpenAIResponsesHandler(BaseTranslation): tool_call_item: Final = _tool_call_output_item_mapping(output_item) if tool_call_item is not None: if tool_calls_to_check is not None: - tool_calls_to_check.append( - tool_call_dict_from_output_item(_scanned_tool_call_item(tool_call_item), output_idx) - ) + tool_calls_to_check.append(_scanned_tool_call(tool_call_item, output_idx)) return summary: Final = _output_compaction_summary(output_item) diff --git a/litellm/proxy/db/spend_log_tool_index.py b/litellm/proxy/db/spend_log_tool_index.py index 591ad81a4e5..92181d546eb 100644 --- a/litellm/proxy/db/spend_log_tool_index.py +++ b/litellm/proxy/db/spend_log_tool_index.py @@ -63,11 +63,26 @@ def response_tool_call_names(completion_response: object) -> tuple[str, ...]: ) return tuple( stripped - for name in chain(called_names, _auto_executed_mcp_tool_names(completion_response)) + for name in chain( + _server_tool_call_names(completion_response), + called_names, + _auto_executed_mcp_tool_names(completion_response), + ) if isinstance(name, str) and (stripped := name.strip()) ) +def _server_tool_call_names(completion_response: object) -> tuple[str | None, ...]: + from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of + from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params + + server_tool_calls: Final = chain.from_iterable( + anthropic_server_tool_call_params(message_field(choice, "message")) + for choice in parts_of(message_field(completion_response, "choices")) + ) + return tuple(server_tool_call["function"].get("name") for server_tool_call in server_tool_calls) + + class _ExecutedTool(BaseModel): model_config = ConfigDict(frozen=True) diff --git a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py index 19b9bccd8aa..3526635a9de 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py @@ -24,6 +24,10 @@ from litellm._logging import verbose_proxy_logger from litellm._version import version as litellm_version from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.core_helpers import iter_items +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_calls, + write_anthropic_server_tool_call_arguments, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, get_ssl_configuration, @@ -515,27 +519,37 @@ class CatoNetworksGuardrail(CustomGuardrail): @staticmethod def _output_fragments(message: Message) -> Sequence[tuple[tuple[str, int | None], str]]: """Assistant text the proxy returns to the caller: ``content`` plus every - ``tool_calls[].function.arguments`` string, each tagged with where a - redaction must be written back. ``content`` is only included when present - so a tool-call-only choice keeps its ``None`` content (the text-vs-tool-call - signal downstream consumers rely on) while its arguments are still inspected.""" - fragments: Final[list] = [] - if message.content is not None: - fragments.append((("content", None), message.content)) - for idx, tool_call in enumerate(message.tool_calls or []): - function = getattr(tool_call, "function", None) - arguments = getattr(function, "arguments", None) - if isinstance(arguments, str) and arguments: - fragments.append((("tool_call", idx), arguments)) - return fragments + tool call's ``function.arguments`` string, Claude's server tool calls first, + each tagged with where a redaction must be written back. ``content`` is only + included when present so a tool-call-only choice keeps its ``None`` content + (the text-vs-tool-call signal downstream consumers rely on) while its + arguments are still inspected.""" + content: Final = () if message.content is None else ((("content", None), message.content),) + return ( + *content, + *CatoNetworksGuardrail._tool_call_fragments("server_tool_call", anthropic_server_tool_calls(message)), + *CatoNetworksGuardrail._tool_call_fragments("tool_call", message.tool_calls or ()), + ) + + @staticmethod + def _tool_call_fragments(kind: str, tool_calls: Sequence[object]) -> tuple[tuple[tuple[str, int | None], str], ...]: + return tuple( + ((kind, idx), arguments) + for idx, arguments in enumerate( + getattr(getattr(tool_call, "function", None), "arguments", None) for tool_call in tool_calls + ) + if isinstance(arguments, str) and arguments + ) @staticmethod def _apply_output_fragment(message: Any, target: tuple[str, int | None], redacted: str) -> None: - kind, idx = target - if kind == "content": - message.content = redacted - else: - message.tool_calls[idx].function.arguments = redacted + match target: + case ("content", _): + message.content = redacted + case ("server_tool_call", int() as idx): + write_anthropic_server_tool_call_arguments(message, idx, redacted) + case (_, idx): + message.tool_calls[idx].function.arguments = redacted @staticmethod def _responses_output_field(item: object, key: str) -> str | Sequence[object] | None: diff --git a/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py b/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py index 3d555707159..8724093a7a3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py @@ -37,6 +37,16 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.litellm_core_utils.core_helpers import iter_items +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + tool_calls_with_server_tool_calls, + with_server_tool_call_arguments, + write_anthropic_server_tool_calls, +) +from litellm.llms.base_llm.guardrail_translation.utils import ( + web_search_call, + with_web_search_call_arguments, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -46,9 +56,12 @@ from litellm.proxy.common_utils.callback_utils import ( add_guardrail_to_applied_guardrails_header, ) from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.llms.anthropic import claude_hosted_web_tool_call from litellm.types.utils import ( Choices, + Delta, LLMResponseTypes, + Message, ModelResponse, ModelResponseStream, TextCompletionResponse, @@ -121,6 +134,12 @@ class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): """Base-class constructor options this guardrail forwards untouched to CustomGuardrail.""" +def _claude_hosted_web_tool_arguments(item: object) -> str | None: + search: Final = web_search_call(item) + hosted_call: Final = None if search is None else claude_hosted_web_tool_call(search) + return None if hosted_call is None else hosted_call[2] + + def _streamed_choice_index(choice: object) -> int: index: Final[object] = getattr(choice, "index", None) return index if isinstance(index, int) else 0 @@ -1654,6 +1673,7 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): @classmethod def _clear_tool_call_arguments(cls, message: object) -> None: """Clear tool-call / function-call arguments after Cisco redaction.""" + cls._clear_server_tool_call_arguments(message) tool_calls = message.get("tool_calls") if isinstance(message, dict) else getattr(message, "tool_calls", None) for tc in tool_calls or []: fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None) @@ -1663,9 +1683,26 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): ) cls._clear_arguments_field(function_call) + @staticmethod + def _clear_server_tool_call_arguments(message: object) -> None: + cleared: Final = tuple( + with_server_tool_call_arguments(server_tool_call, "{}") + for server_tool_call in anthropic_server_tool_call_params(message) + ) + match message: + case Message() | Delta(): + write_anthropic_server_tool_calls(message, cleared) + case dict() if cleared: + message["provider_specific_fields"] = { # rebind-ok: a redaction must land on the message it scanned + **(message.get("provider_specific_fields") or {}), + "server_tool_calls": list(cleared), + } + case _: + pass + def _redact_responses_api_output( self, - output_items: list, + output_items: list[object], sanitized_text: str | None, sanitized_messages: list[dict[str, object]] | None, ) -> bool: @@ -1677,7 +1714,10 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): if not replacement_text: return False applied = False - for item in output_items: + for index, item in enumerate(output_items): + if _claude_hosted_web_tool_arguments(item): + output_items[index] = with_web_search_call_arguments(item, "{}") # rebind-ok: in-place redaction + applied = True content = getattr(item, "content", None) or (item.get("content") if isinstance(item, dict) else None) if isinstance(content, list): for part in content: @@ -2038,8 +2078,9 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): output_items: Final = getattr(response, "output", None) if not isinstance(output_items, list): return [] + typed_output_items: Final[Sequence[object]] = output_items output_parts: Final[list[str]] = [] - for item in output_items: + for item in typed_output_items: get = item.get if isinstance(item, dict) else (lambda k: getattr(item, k, None)) for part in get("content") or []: pget = part.get if isinstance(part, dict) else (lambda k: getattr(part, k, None)) @@ -2053,6 +2094,8 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): direct = get("text") if isinstance(direct, str) and direct: output_parts.append(direct) + if hosted_arguments := _claude_hosted_web_tool_arguments(item): + output_parts.append(hosted_arguments) joined = " ".join(output_parts) return [{"role": "assistant", "content": joined}] if joined else [] @@ -2095,8 +2138,7 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): @classmethod def _extract_message_tool_argument_parts(cls, message: object) -> list[str]: parts: Final[list[str]] = [] - tool_calls = message.get("tool_calls") if isinstance(message, dict) else getattr(message, "tool_calls", None) - for tool_call in tool_calls or []: + for tool_call in tool_calls_with_server_tool_calls(message): args = cls._extract_tool_call_arguments(tool_call) if args: parts.append(args) diff --git a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py index 97b35769f63..5902a07dea2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py @@ -40,6 +40,12 @@ from litellm.integrations.custom_guardrail import ( ) from litellm.integrations.custom_guardrail import dc as global_cache from litellm.litellm_core_utils.core_helpers import iter_items +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + anthropic_server_tool_calls, + with_server_tool_call_arguments, + write_anthropic_server_tool_calls, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -50,6 +56,7 @@ from litellm.proxy.guardrails._content_utils import ( has_non_string_content, ) from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.llms.openai import ChatCompletionAssistantToolCall class LassoResponse(TypedDict): @@ -259,7 +266,7 @@ class LassoGuardrail(CustomGuardrail): msg = choice.message if msg.content: response_messages.append({"role": "assistant", "content": msg.content}) - for call in getattr(msg, "tool_calls", None) or []: + for call in (*anthropic_server_tool_calls(msg), *(getattr(msg, "tool_calls", None) or ())): call_id, name, input_data = self._extract_tool_call_fields(call) if not call_id or not name: continue @@ -962,6 +969,14 @@ class LassoGuardrail(CustomGuardrail): text_cursor += 1 verbose_proxy_logger.debug("Applied masked text content to choice %s", text_cursor) + write_anthropic_server_tool_calls( + msg, + tuple( + self._masked_server_tool_call(server_tool_call, masked_tool_use) + for server_tool_call in anthropic_server_tool_call_params(msg) + ), + ) + for call in getattr(msg, "tool_calls", None) or []: call_id = self._get_field(call, "id") if call_id and call_id in masked_tool_use: @@ -977,6 +992,16 @@ class LassoGuardrail(CustomGuardrail): func.arguments = json.dumps(masked_input) verbose_proxy_logger.debug("Applied masked tool_call arguments for call_id=%s", call_id) + @staticmethod + def _masked_server_tool_call( + server_tool_call: ChatCompletionAssistantToolCall, + masked_tool_use: Mapping[object, Mapping[str, object]], + ) -> ChatCompletionAssistantToolCall: + masked_input: Final = masked_tool_use.get(server_tool_call["id"], {}).get("input") + if masked_input is None: + return server_tool_call + return with_server_tool_call_arguments(server_tool_call, json.dumps(masked_input)) + @staticmethod def get_config_model() -> type["GuardrailConfigModel"] | None: from litellm.types.proxy.guardrails.guardrail_hooks.lasso import ( diff --git a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py index d2dafb04fe0..5e7a0240545 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py +++ b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py @@ -24,13 +24,17 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.litellm_core_utils.core_helpers import attribute_items, attribute_or_none, iter_items, require_attributes +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + write_anthropic_server_tool_call_arguments, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) from litellm.proxy._types import UserAPIKeyAuth from litellm.types.guardrails import GuardrailEventHooks -from litellm.types.utils import GenericGuardrailAPIInputs, TextChoices +from litellm.types.utils import GenericGuardrailAPIInputs, Message, TextChoices if TYPE_CHECKING: from litellm.caching.caching import DualCache @@ -87,6 +91,19 @@ _DEFAULT_TIMEOUT_SECONDS: Final = 10.0 _SHIELD_BODY: Final = TypeAdapter(dict[str, object]) +def _server_tool_call_slots(message: object) -> tuple[Slot, ...]: + if not isinstance(message, Message): + return () + return tuple( + (arguments, functools.partial(write_anthropic_server_tool_call_arguments, message, index)) + for index, arguments in enumerate( + server_tool_call["function"].get("arguments") + for server_tool_call in anthropic_server_tool_call_params(message) + ) + if isinstance(arguments, str) and arguments + ) + + def _string_tool_arguments(calls: Sequence[object]) -> Iterator[tuple[object, str]]: for call in calls: function = read_field(call, "function") @@ -382,6 +399,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail): content = getattr(message, "content", None) if isinstance(content, str) and content: pending.append((content, functools.partial(setattr, message, "content"))) + pending.extend(_server_tool_call_slots(message)) for tool_call in attribute_items(message, "tool_calls"): function: object = getattr(tool_call, "function", None) arguments = getattr(function, "arguments", None) if function is not None else None diff --git a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py index f83314af548..1956170ba67 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py +++ b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py @@ -13,6 +13,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, ) from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -434,9 +435,7 @@ class PurviewGuardrailBase: args: Final[list[str]] = [] # tool_calls: [{"function": {"arguments": "..."}}] - tool_calls: Final[Sequence[object] | None] = ( - message.get("tool_calls") if isinstance(message, dict) else getattr(message, "tool_calls", None) - ) + tool_calls: Final = tool_calls_with_server_tool_calls(message) if tool_calls: for tc in tool_calls: fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None) diff --git a/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py b/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py index 2194238cea0..4ec3ed21359 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py +++ b/litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py @@ -1,5 +1,6 @@ # litellm/proxy/guardrails/guardrail_hooks/pangea.py import os +from collections.abc import Callable from typing import TYPE_CHECKING, Final from fastapi import HTTPException @@ -10,6 +11,11 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.llms.anthropic.common_utils import ( + message_dict_with_server_tool_calls, + message_dict_with_server_tool_calls_split_out, + validated_object_mapping, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -30,6 +36,13 @@ if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +def _choice_with_message(choice: object, rewrite: Callable[[object], object]) -> object: + mapping: Final = validated_object_mapping(choice) + if mapping is None or "message" not in mapping: + return choice + return {**mapping, "message": rewrite(mapping["message"])} + + class PangeaGuardrailMissingSecrets(Exception): """Custom exception for missing Pangea secrets.""" @@ -242,9 +255,13 @@ class PangeaHandler(CustomGuardrail): for c in choices: if isinstance(c, Choices): try: - serialized_choices.append(c.model_dump()) + serialized_choices.append( + _choice_with_message(c.model_dump(), message_dict_with_server_tool_calls) + ) except Exception: - serialized_choices.append(c.dict()) + serialized_choices.append( + _choice_with_message(c.dict(), message_dict_with_server_tool_calls) + ) else: serialized_choices.append(c) choices = serialized_choices @@ -269,7 +286,9 @@ class PangeaHandler(CustomGuardrail): return output: Final = ai_guard_response.get("result", {}).get("output", {}) - response.choices = output["choices"] + response.choices = [ # pyright: ignore[reportAttributeAccessIssue] # Pangea returns untyped choice dicts + _choice_with_message(choice, message_dict_with_server_tool_calls_split_out) for choice in output["choices"] + ] return response @log_guardrail_information diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index 3cd9dc6a7b4..379c2c4b075 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -27,6 +27,12 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + tool_calls_with_server_tool_calls, + with_server_tool_call_arguments, + write_anthropic_server_tool_calls, +) from litellm.llms.base_llm.guardrail_translation.utils import ( effective_scan_only_tool_results_for_guardrail, effective_skip_system_message_for_guardrail, @@ -364,10 +370,9 @@ class PanwPrismaAirsHandler(CustomGuardrail): text_parts.append(str(choice.message.content)) # Extract tool call arguments - if hasattr(choice.message, "tool_calls") and choice.message.tool_calls: - for tool_call in choice.message.tool_calls: - if hasattr(tool_call, "function") and hasattr(tool_call.function, "arguments"): - text_parts.append(str(tool_call.function.arguments)) + for tool_call in tool_calls_with_server_tool_calls(choice.message): + if hasattr(tool_call, "function") and hasattr(tool_call.function, "arguments"): + text_parts.append(str(tool_call.function.arguments)) # Extract function call arguments (legacy) if hasattr(choice.message, "function_call") and choice.message.function_call: @@ -756,6 +761,14 @@ class PanwPrismaAirsHandler(CustomGuardrail): elif isinstance(content, list): choice.message.content = self._mask_content_list(content, masked_text) + write_anthropic_server_tool_calls( + choice.message, + tuple( + with_server_tool_call_arguments(server_tool_call, masked_text) + for server_tool_call in anthropic_server_tool_call_params(choice.message) + ), + ) + # Mask tool call arguments if hasattr(choice.message, "tool_calls") and choice.message.tool_calls: for tool_call in choice.message.tool_calls: diff --git a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py index 3f025cfc8b3..0facab6ddef 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py +++ b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py @@ -26,6 +26,7 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.anthropic.common_utils import message_dict_with_server_tool_calls from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -430,7 +431,9 @@ class PillarGuardrail(CustomGuardrail): # Extract response messages in the format Pillar expects response_dict = response.model_dump() if hasattr(response, "model_dump") else {} response_messages: Final = [ - choice.get("message") for choice in response_dict.get("choices", []) if choice.get("message") + message_dict_with_server_tool_calls(choice["message"]) + for choice in response_dict.get("choices", []) + if choice.get("message") ] if not response_messages: diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 7e2f6bfec33..8d0f71b554c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -12,7 +12,7 @@ import asyncio import json import re import threading -from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator, Awaitable, Iterator, Sequence +from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator, Awaitable, Callable, Iterator, Sequence from contextlib import asynccontextmanager from dataclasses import dataclass from datetime import datetime @@ -43,6 +43,11 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + with_server_tool_call_arguments, + write_anthropic_server_tool_calls, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames from litellm.proxy.guardrails.anthropic_sse import ( @@ -1392,6 +1397,18 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): verbose_proxy_logger.debug("No pii_tokens found in request_data['metadata'] — nothing to unmask") presidio_config: Final = self.get_presidio_settings_from_request_data(request_data or {}) + async def pii_processed_text(text: str) -> str: + match mode: + case "unmask": + return self._unmask_pii_text(text, pii_tokens) + case "mask": + return await self.check_pii( + text=text, + output_parse_pii=False, + presidio_config=presidio_config, + request_data=request_data, + ) + for choice in response.choices: message: Message | None = getattr(choice, "message", None) if message is None: @@ -1400,15 +1417,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): # 1. Process content content = getattr(message, "content", None) if isinstance(content, str): - if mode == "unmask": - message.content = self._unmask_pii_text(content, pii_tokens) - elif mode == "mask": - message.content = await self.check_pii( - text=content, - output_parse_pii=False, - presidio_config=presidio_config, - request_data=request_data, - ) + message.content = await pii_processed_text(content) elif isinstance(content, list): for item in content: if not isinstance(item, dict): @@ -1416,54 +1425,43 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): text_value = item.get("text") if text_value is None: continue - if mode == "unmask": - item["text"] = self._unmask_pii_text(text_value, pii_tokens) - elif mode == "mask": - item["text"] = await self.check_pii( - text=text_value, - output_parse_pii=False, - presidio_config=presidio_config, - request_data=request_data, - ) + item["text"] = await pii_processed_text(text_value) + + await self._process_server_tool_calls_for_pii(message, pii_processed_text) # 2. Process tool calls for tool_call in attribute_items(message, "tool_calls"): function = attribute_or_none(tool_call, "function") args = attribute_or_none(function, "arguments") if function else None if isinstance(args, str): - if mode == "unmask": - _write_attribute(function, "arguments", self._unmask_pii_text(args, pii_tokens)) - elif mode == "mask": - _write_attribute( - function, - "arguments", - await self.check_pii( - text=args, - output_parse_pii=False, - presidio_config=presidio_config, - request_data=request_data, - ), - ) + _write_attribute(function, "arguments", await pii_processed_text(args)) # 3. Process legacy function calls function_call = attribute_or_none(message, "function_call") legacy_args = attribute_or_none(function_call, "arguments") if function_call else None if isinstance(legacy_args, str): - if mode == "unmask": - _write_attribute(function_call, "arguments", self._unmask_pii_text(legacy_args, pii_tokens)) - elif mode == "mask": - _write_attribute( - function_call, - "arguments", - await self.check_pii( - text=legacy_args, - output_parse_pii=False, - presidio_config=presidio_config, - request_data=request_data, - ), - ) + _write_attribute(function_call, "arguments", await pii_processed_text(legacy_args)) return response + @staticmethod + async def _process_server_tool_calls_for_pii( + message: Message, + pii_processed_text: Callable[[str], Awaitable[str]], + ) -> None: + server_tool_calls: Final = anthropic_server_tool_call_params(message) + if not server_tool_calls: + return + processed_arguments: Final = tuple( + [ + await pii_processed_text(server_tool_call["function"].get("arguments", "")) + for server_tool_call in server_tool_calls + ] + ) + write_anthropic_server_tool_calls( + message, + tuple(map(with_server_tool_call_arguments, server_tool_calls, processed_arguments)), + ) + async def _mask_output_response( self, response: ModelResponse | EmbeddingResponse | ImageResponse, diff --git a/litellm/proxy/guardrails/guardrail_hooks/repelloai/repelloai.py b/litellm/proxy/guardrails/guardrail_hooks/repelloai/repelloai.py index b1f0f588ade..2e1ac62d850 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/repelloai/repelloai.py +++ b/litellm/proxy/guardrails/guardrail_hooks/repelloai/repelloai.py @@ -12,6 +12,7 @@ from pydantic import BaseModel, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] httpxSpecialProvider, @@ -80,13 +81,11 @@ class RepelloAIGuardrail(CustomGuardrail): def _extract_tool_call_args_from_message(cls, message: object) -> list[str]: args: Final[list[str]] = [] - tool_calls: Final = cls._get_field(message, "tool_calls") - if _is_object_list(tool_calls): - for tool_call in tool_calls: - function = cls._get_field(tool_call, "function") - arguments = cls._get_field(function, "arguments") - if isinstance(arguments, str) and arguments.strip(): - args.append(arguments) + for tool_call in tool_calls_with_server_tool_calls(message): + function = cls._get_field(tool_call, "function") + arguments = cls._get_field(function, "arguments") + if isinstance(arguments, str) and arguments.strip(): + args.append(arguments) function_call: Final = cls._get_field(message, "function_call") arguments = cls._get_field(function_call, "arguments") diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py index e8c094c0c37..dceb0b6aeae 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -19,6 +19,7 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params, anthropic_server_tool_calls from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import ( add_guardrail_to_applied_guardrails_header, @@ -516,7 +517,7 @@ class ToolPermissionGuardrail(CustomGuardrail): for choice_index, choice in enumerate(response.choices): if isinstance(choice, Choices): - for tool in choice.message.tool_calls or []: + for tool in (*(choice.message.tool_calls or ()), *anthropic_server_tool_calls(choice.message)): tool_calls.append(tool) legacy_tool_call = self._legacy_function_call_to_tool_call( getattr(choice.message, "function_call", None), choice_index @@ -791,6 +792,23 @@ class ToolPermissionGuardrail(CustomGuardrail): choice.message.tool_calls = filtered_tool_calls if filtered_tool_calls else None + server_error_results = tuple( + error_results.get(self._judged_tool_call(server_tool_call)) + for server_tool_call in anthropic_server_tool_calls(choice.message) + ) + if any(server_error_results): + error_messages.extend(result.content for result in server_error_results if result is not None) + choice.message.provider_specific_fields = { + **(choice.message.provider_specific_fields or {}), + "server_tool_calls": [ + server_tool_call + for server_tool_call, result in zip( + anthropic_server_tool_call_params(choice.message), server_error_results + ) + if result is None + ], + } + legacy_tool_call = self._legacy_function_call_to_tool_call( getattr(choice.message, "function_call", None), choice_index ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index 5ce2dfb3b87..6f6d056423f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -25,6 +25,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket from litellm.llms import get_guardrail_translation_mapping, load_guardrail_translation_mappings +from litellm.llms.anthropic.common_utils import anthropic_server_call_fields from litellm.proxy._types import UserAPIKeyAuth from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import ( @@ -144,6 +145,15 @@ def _ends_stream(endpoint_translation: object, item: object) -> bool: return has_ended is not None and has_ended((item,)) +def _passthrough_delta(delta: object) -> Delta: + server_call_fields: Final = anthropic_server_call_fields(delta) + role: Final = getattr(delta, "role", None) + tool_calls: Final = getattr(delta, "tool_calls", None) + if not server_call_fields: + return Delta(content=None, role=role, tool_calls=tool_calls) + return Delta(content=None, role=role, tool_calls=tool_calls, provider_specific_fields=dict(server_call_fields)) + + def _held_choices(held_chars_per_choice: Mapping[int, int]) -> frozenset[int]: return frozenset(idx for idx, held in held_chars_per_choice.items() if held > 0) @@ -974,7 +984,7 @@ class UnifiedLLMGuardrails(CustomLogger): def _chunk_has_tool_calls(item: object) -> bool: for choice in _chunk_choices(item): delta = getattr(choice, "delta", None) - if getattr(delta, "tool_calls", None): + if getattr(delta, "tool_calls", None) or anthropic_server_call_fields(delta): return True return False @@ -1042,11 +1052,7 @@ class UnifiedLLMGuardrails(CustomLogger): synthetic_choices.append( StreamingChoices( index=idx, - delta=Delta( - content=None, - role=getattr(delta, "role", None), - tool_calls=getattr(delta, "tool_calls", None), - ), + delta=_passthrough_delta(delta), finish_reason=passthrough_finish, ) ) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 892da541935..3e053dd87a9 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -9,6 +9,7 @@ import litellm from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend +from litellm.llms.anthropic.common_utils import anthropic_server_tool_calls from litellm.main import stream_chunk_builder from litellm.responses.litellm_completion_transformation.compaction_items import compaction_output_item from litellm.responses.litellm_completion_transformation.custom_tools import ( @@ -486,13 +487,16 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): try: message: Final = litellm_complete_object.choices[0].message - tool_calls = getattr(message, "tool_calls", None) + server_tool_calls: Final = anthropic_server_tool_calls(message) + tool_calls: Final = [*server_tool_calls, *(getattr(message, "tool_calls", None) or ())] except Exception: - tool_calls = None - - if not tool_calls or not isinstance(tool_calls, list): return + if not tool_calls: + return + + server_tool_call_ids: Final = frozenset(tool_call.id for tool_call in server_tool_calls) + for tc in tool_calls: call_id_raw = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None) if not call_id_raw: @@ -518,6 +522,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if (search_execution := self._tool_search_execution(call_id, fn_name)) is not None: self._queue_tool_search_call_done(call_id, output_index, fn_args, search_execution) continue + if call_id in server_tool_call_ids: + continue # Track if this is a new tool call that wasn't streamed is_new_tool_call = call_id not in self._tool_item_id_by_call_id @@ -592,10 +598,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): } ), ) + finished_events: Final = ( + () + if item.status == "in_progress" + else ((ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),) + ) for event_type, event_class in ( (ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, WebSearchCallInProgressEvent), (ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, WebSearchCallSearchingEvent), - (ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent), + *finished_events, ): self._sequence_number += 1 event = event_class(type=event_type, output_index=output_index, item_id=item.id) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 8f0a85d67de..83f46651e50 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -56,7 +56,12 @@ from litellm.litellm_core_utils.hidden_params import ( set_hidden_params, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + anthropic_server_tool_calls, + supports_anthropic_cache_control, + validated_object_mapping, +) from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support from litellm.llms.openrouter.chat.transformation import openrouter_model_supports_cache_control @@ -66,11 +71,12 @@ from litellm.responses.litellm_completion_transformation.session_handler import ResponsesSessionHandler, StoredTurns, ) -from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex +from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex, claude_hosted_web_tool_call from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, + ChatCompletionAssistantToolCall, ChatCompletionAudioObject, ChatCompletionCachedContent, ChatCompletionFileObject, @@ -875,6 +881,22 @@ def _chain_session_metadata(chain_session: str, request_kwargs: Mapping[str, obj return {key: {**_STR_KEY_DICT_ADAPTER.validate_python(metadata), CALLER_SESSION_ID_METADATA_KEY: chain_session}} +def claude_web_search_call_tool_call(search: ResponseFunctionWebSearch) -> ChatCompletionAssistantToolCall | None: + claude_call: Final = claude_hosted_web_tool_call(search) + if claude_call is None: + return None + tool_id, name, arguments = claude_call + return ChatCompletionAssistantToolCall( + id=tool_id, + type="function", + function=ChatCompletionToolCallFunctionChunk(name=name, arguments=arguments), + ) + + +def _pending_claude_server_tool_call(search: ResponseFunctionWebSearch) -> ChatCompletionAssistantToolCall | None: + return claude_web_search_call_tool_call(search) if search.status == "in_progress" else None + + class LiteLLMCompletionResponsesConfig: @staticmethod def get_supported_openai_params(model: str) -> list: @@ -1780,10 +1802,22 @@ class LiteLLMCompletionResponsesConfig: message_cache_control: Final = ( None if isinstance(content, list) else new_cache_control or previous_cache_control ) + new_server_tool_calls: Final = anthropic_server_tool_call_params(new_message) + server_tool_call_fields: Final = ( + { + "provider_specific_fields": { + **(validated_object_mapping(last_message.get("provider_specific_fields")) or {}), + "server_tool_calls": [*anthropic_server_tool_call_params(last_message), *new_server_tool_calls], + } + } + if new_server_tool_calls + else {} + ) merged: Final = { **{key: value for key, value in last_message.items() if key != "cache_control"}, "content": content, **({} if message_cache_control is None else {"cache_control": message_cache_control}), + **server_tool_call_fields, } return cast( # cast-ok: preserves the assistant fields and content blocks ChatCompletionAssistantMessage, merged @@ -2225,10 +2259,15 @@ class LiteLLMCompletionResponsesConfig: content="Hosted web search: " + replay.model_dump_json(exclude_none=True), ) ] + hosted_search_text: Final = "Hosted web search: " + search.model_dump_json(exclude_none=True) + pending_server_tool_call: Final = _pending_claude_server_tool_call(search) + if pending_server_tool_call is None: + return [GenericChatCompletionMessage(role="assistant", content=hosted_search_text)] return [ - GenericChatCompletionMessage( + ChatCompletionResponseMessage( role="assistant", - content="Hosted web search: " + search.model_dump_json(exclude_none=True), + content=hosted_search_text, + provider_specific_fields={"server_tool_calls": [pending_server_tool_call]}, ) ] if LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(input_item): @@ -2958,13 +2997,12 @@ class LiteLLMCompletionResponsesConfig: all_chat_completion_tools: Final[list[ChatCompletionMessageToolCall]] = [] for choice in chat_completion_response.choices: if isinstance(choice, Choices): - if choice.message.tool_calls: - all_chat_completion_tools.extend(choice.message.tool_calls) - for tool_call in choice.message.tool_calls: - TOOL_CALLS_CACHE.set_cache( - key=tool_call.id, - value=tool_call, - ) + for tool_call in (*anthropic_server_tool_calls(choice.message), *(choice.message.tool_calls or ())): + all_chat_completion_tools.append(tool_call) + TOOL_CALLS_CACHE.set_cache( + key=tool_call.id, + value=tool_call, + ) request_tools: Final = responses_api_request.get("tools") if responses_api_request is not None else None custom_tool_names: Final = extract_custom_tool_names(request_tools) @@ -3381,7 +3419,22 @@ class LiteLLMCompletionResponsesConfig: for item in responses_output ] - return responses_output + server_tool_call_ids: Final = LiteLLMCompletionResponsesConfig._server_tool_call_ids(choices) + return [ + item + for item in responses_output + if not ( + getattr(item, "type", None) == "function_call" + and getattr(item, "call_id", None) in server_tool_call_ids + ) + ] + + @staticmethod + def _server_tool_call_ids(choices: Sequence[Choices]) -> frozenset[str]: + server_tool_calls: Final = chain.from_iterable( + anthropic_server_tool_calls(choice.message) for choice in choices + ) + return frozenset(tool_call.id for tool_call in server_tool_calls) @staticmethod def _extract_tool_result_output_items( diff --git a/litellm/tool_loop.py b/litellm/tool_loop.py index d0b2850bb5b..554fdff4fce 100644 --- a/litellm/tool_loop.py +++ b/litellm/tool_loop.py @@ -120,9 +120,12 @@ def _replays_provider_specific_fields( client_tool_calls: tuple[ChatCompletionMessageToolCall, ...], ) -> bool: from litellm.completion_extras.litellm_responses_transformation.transformation import carries_tool_search_items + from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params - return client_tool_calls != tool_calls or carries_tool_search_items( - getattr(message, "provider_specific_fields", None) + return ( + client_tool_calls != tool_calls + or bool(anthropic_server_tool_call_params(message)) + or carries_tool_search_items(getattr(message, "provider_specific_fields", None)) ) diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 152614e4385..4992d0e780e 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -2,10 +2,12 @@ from collections.abc import Iterable, Mapping, Sequence from enum import Enum from typing import Final, Literal, TypeAlias +from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch from pydantic import ConfigDict from typing_extensions import NotRequired, ReadOnly, Required, TypedDict from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.responses.main import hosted_web_tool_call, web_search_call_tool_id from .openai import ( ChatCompletionCachedContent, @@ -858,6 +860,15 @@ ANTHROPIC_TOOL_SEARCH_TOOL_TYPES: Final = frozenset( ANTHROPIC_TOOL_SEARCH_TOOL_NAMES: Final = frozenset({"tool_search_tool_regex", "tool_search_tool_bm25"}) ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX: Final = "srvtoolu_" + +def claude_hosted_web_tool_call(search: ResponseFunctionWebSearch) -> tuple[str, str, str] | None: + tool_id: Final = web_search_call_tool_id(search.id) + if tool_id is None or not tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX): + return None + hosted_call: Final = hosted_web_tool_call(search.action) + return None if hosted_call is None else (tool_id, *hosted_call) + + # Effort beta header constant ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24" diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 72dc63a581f..44f50a9df9c 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -1,10 +1,17 @@ import builtins +import json from collections.abc import Mapping, Sequence from typing import Final, Literal, Optional, Union from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall -from openai.types.responses.response_function_web_search import ActionSearchSource, ResponseFunctionWebSearch -from pydantic import ConfigDict, PrivateAttr +from openai.types.responses.response_function_web_search import ( + Action, + ActionOpenPage, + ActionSearch, + ActionSearchSource, + ResponseFunctionWebSearch, +) +from pydantic import ConfigDict, PrivateAttr, TypeAdapter, ValidationError from typing_extensions import TypedDict from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject @@ -42,14 +49,11 @@ class OutputFunctionToolCall(BaseLiteLLMOpenAIResponseObject): phase: Phase = None -def build_web_search_call( - tool_id: str, - tool_input: object, - result: object, - status: Literal["in_progress", "searching", "completed", "failed"] | None = None, -) -> ResponseFunctionWebSearch: +HOSTED_WEB_TOOL_NAMES: Final = frozenset({"web_search", "web_fetch"}) + + +def _web_search_action(tool_input: object, content: object) -> ActionSearch: query: Final = tool_input.get("query", "") if isinstance(tool_input, Mapping) else "" - content: Final = result.get("content") if isinstance(result, Mapping) else None result_items: Final = content if isinstance(content, Sequence) and not isinstance(content, (str, bytes)) else () sources: Final = [ ActionSearchSource(type="url", url=url) @@ -58,20 +62,96 @@ def build_web_search_call( and item.get("type") == "web_search_result" and isinstance((url := item.get("url")), str) ] - failed: Final = isinstance(content, Mapping) and content.get("type") == "web_search_tool_result_error" - return ResponseFunctionWebSearch( - id=f"ws_{tool_id}", - type="web_search_call", - status=status or ("failed" if failed else "completed"), - action={ - "type": "search", - "query": query if isinstance(query, str) else "", - "queries": [query] if isinstance(query, str) and query else [], - "sources": sources, - }, + return ActionSearch( + type="search", + query=query if isinstance(query, str) else "", + queries=[query] if isinstance(query, str) and query else [], + sources=sources, ) +def _web_fetch_action(tool_input: object) -> ActionOpenPage: + match tool_input: + case {"url": str() as url}: + return ActionOpenPage(type="open_page", url=url) + case _: + return ActionOpenPage(type="open_page", url=None) + + +def _result_content(result: object) -> object: + match result: + case {"content": object() as content}: + return content + case _: + return None + + +def _is_tool_result_error(content: object, tool_name: str) -> bool: + match content: + case {"type": str() as content_type}: + return content_type == f"{tool_name}_tool_result_error" + case _: + return False + + +WEB_SEARCH_CALL_ID_PREFIX: Final = "ws_" + + +def build_web_search_call( + tool_id: str, + tool_input: object, + result: object, + status: Literal["in_progress", "searching", "completed", "failed"] | None = None, + tool_name: str = "web_search", +) -> ResponseFunctionWebSearch: + content: Final = _result_content(result) + return ResponseFunctionWebSearch( + id=f"{WEB_SEARCH_CALL_ID_PREFIX}{tool_id}", + type="web_search_call", + status=status or ("failed" if _is_tool_result_error(content, tool_name) else "completed"), + action=_web_fetch_action(tool_input) if tool_name == "web_fetch" else _web_search_action(tool_input, content), + ) + + +def web_search_call_tool_id(call_id: str) -> str | None: + tool_id: Final = call_id.removeprefix(WEB_SEARCH_CALL_ID_PREFIX) + return None if tool_id == call_id else tool_id + + +_HOSTED_WEB_TOOL_ARGUMENTS: Final = TypeAdapter(dict[str, object]) + + +def hosted_web_tool_call(action: object) -> tuple[str, str] | None: + match action: + case ActionSearch(query=str() as query): + return "web_search", json.dumps({"query": query}) + case ActionOpenPage(url=str() as url): + return "web_fetch", json.dumps({"url": url}) + case _: + return None + + +def _decoded_hosted_web_tool_arguments(arguments: str) -> Mapping[str, object]: + try: + return _HOSTED_WEB_TOOL_ARGUMENTS.validate_json(arguments) + except ValidationError: + return {} + + +def with_hosted_web_tool_arguments(action: Action, arguments: str) -> Action: + match action, _decoded_hosted_web_tool_arguments(arguments): + case ActionSearch(), {"query": str() as query}: + return action.model_copy(update={"query": query, "queries": [query] if query else []}) + case ActionSearch(), _: + return action.model_copy(update={"query": "", "queries": []}) + case ActionOpenPage(), {"url": str() as url}: + return action.model_copy(update={"url": url}) + case ActionOpenPage(), _: + return action.model_copy(update={"url": None}) + case _: + return action + + class OutputImageGenerationCall(BaseLiteLLMOpenAIResponseObject): """An image generation call output""" diff --git a/tests/unit/integrations/arize/test_arize_utils.py b/tests/unit/integrations/arize/test_arize_utils.py index 31699359bbe..db7144b125a 100644 --- a/tests/unit/integrations/arize/test_arize_utils.py +++ b/tests/unit/integrations/arize/test_arize_utils.py @@ -725,6 +725,73 @@ def test_arize_output_value_falls_back_to_tool_calls_summary(): assert "SF" in out +def test_arize_server_tool_calls_lead_the_output_summary_and_tool_call_attributes() -> None: + span: Final = TracerProvider().get_tracer(__name__).start_span("litellm_request") + kwargs: Final = { + "model": "claude-opus-5-5", + "messages": [{"role": "user", "content": "persimmon facts?"}], + "standard_logging_object": {"model_parameters": {}, "metadata": {}, "call_type": "completion"}, + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "anthropic"}, + } + response_obj: Final = ModelResponse( + usage={"total_tokens": 10, "completion_tokens": 4, "prompt_tokens": 6}, + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=litellm.Message( + role="assistant", + content=None, + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "SF"}'}, + } + ], + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + ] + }, + ), + ) + ], + model="claude-opus-5-5", + id="r-server-tools", + ) + + ArizeLogger.set_arize_attributes(span, kwargs, response_obj) + + tool_calls_prefix: Final = f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_TOOL_CALLS}." + tool_call_attributes: Final = { + key: value for key, value in span.attributes.items() if key.startswith(tool_calls_prefix) + } + assert json.loads(span.attributes[SpanAttributes.OUTPUT_VALUE]) == { + "tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + }, + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": '{"city": "SF"}'}}, + ] + } + assert tool_call_attributes == { + f"{tool_calls_prefix}0.{ToolCallAttributes.TOOL_CALL_ID}": "srvtoolu_01", + f"{tool_calls_prefix}0.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}": "web_search", + f"{tool_calls_prefix}0.{ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON}": '{"query": "persimmon"}', + f"{tool_calls_prefix}1.{ToolCallAttributes.TOOL_CALL_ID}": "call_1", + f"{tool_calls_prefix}1.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}": "get_weather", + f"{tool_calls_prefix}1.{ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON}": '{"city": "SF"}', + } + + def test_arize_output_value_unchanged_when_content_present(): """Regression guard: when content is non-empty, OUTPUT_VALUE must be exactly that content (no summary written).""" diff --git a/tests/unit/integrations/datadog/test_datadog_llm_obs.py b/tests/unit/integrations/datadog/test_datadog_llm_obs.py index a7826e11483..dcc1567f5f8 100644 --- a/tests/unit/integrations/datadog/test_datadog_llm_obs.py +++ b/tests/unit/integrations/datadog/test_datadog_llm_obs.py @@ -110,6 +110,41 @@ def test_output_tool_calls_use_the_datadog_tool_call_schema(logger: DataDogLLMOb assert "function" not in message["tool_calls"][0] + +def test_claude_server_tool_calls_are_reported_before_client_tool_calls(logger: DataDogLLMObsLogger) -> None: + payload = build( + logger, + response_message={ + "role": "assistant", + "content": None, + "tool_calls": [ASSISTANT_TOOL_CALL], + "provider_specific_fields": { + "server_tool_calls": [ + { + "id": "srvtoolu_01Search", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "paris weather"}'}, + } + ] + }, + }, + ) + + assert payload["meta"]["output"]["messages"][0]["tool_calls"] == [ + { + "name": "web_search", + "arguments": {"query": "paris weather"}, + "tool_id": "srvtoolu_01Search", + "type": "function", + }, + { + "name": "get_weather", + "arguments": {"city": "Paris", "unit": "c"}, + "tool_id": "call_abc123", + "type": "function", + }, + ] + def test_tool_call_identifiers_that_are_not_strings_are_blanked_not_stringified(logger: DataDogLLMObsLogger) -> None: payload = build( logger, diff --git a/tests/unit/integrations/newrelic/test_newrelic.py b/tests/unit/integrations/newrelic/test_newrelic.py index d09732c6eb2..79dde4f5959 100644 --- a/tests/unit/integrations/newrelic/test_newrelic.py +++ b/tests/unit/integrations/newrelic/test_newrelic.py @@ -1,3 +1,4 @@ +import json import os import sys from datetime import datetime, timezone @@ -45,6 +46,12 @@ NR_ENV = { "NEW_RELIC_APP_NAME": "test-app", } +_SERVER_CALL: Final = { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, +} + def make_logger(**kwargs) -> NewRelicLogger: """Instantiate NewRelicLogger with NR agent calls mocked out.""" @@ -361,6 +368,19 @@ class TestExtractMessageContent: assert "get_weather" in result assert "call_1" in result + @pytest.mark.parametrize( + "message", + [ + {"role": "assistant", "content": None, "provider_specific_fields": {"server_tool_calls": [_SERVER_CALL]}}, + litellm.Message( + role="assistant", content=None, provider_specific_fields={"server_tool_calls": [_SERVER_CALL]} + ), + ], + ids=["dict", "message"], + ) + def test_server_tool_call_only_message_serializes_the_server_call(self, message): + assert json.loads(self.logger._extract_message_content(message)) == [_SERVER_CALL] + def test_multimodal_list_serialized_as_json(self): msg = { "content": [ diff --git a/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py index 2d0b4701c9f..961bfcd3867 100644 --- a/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py @@ -48,6 +48,7 @@ from litellm.integrations.otel.model.spans import ( validate_registry, ) from litellm.integrations.otel.model.trace_controls import TraceControls, caller_trace_controls +from litellm.types.responses.main import build_web_search_call @pytest.fixture(autouse=True) @@ -798,6 +799,37 @@ def test_responses_tool_calls_fold_into_the_assistant_message_with_tool_calls_fi assert data.finish_reasons == ("tool_calls",) +def test_responses_claude_web_calls_lead_the_tool_calls_without_ending_the_turn_on_tool_calls(): + claude_fetch = build_web_search_call("srvtoolu_01", {"url": "https://example.com/"}, None, tool_name="web_fetch") + openai_search = { + "type": "web_search_call", + "id": "ws_1", + "status": "completed", + "action": {"type": "search", "query": "news"}, + } + client_call = {"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": "{}"} + claude_tool_call = { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_fetch", "arguments": '{"url": "https://example.com/"}'}, + } + + hosted_only = LLMCallSpanData.from_standard_logging_payload( + _responses_payload([_RESPONSES_TEXT_ITEM, claude_fetch.model_dump(), openai_search]), capture_content=True + ) + with_client_call = LLMCallSpanData.from_standard_logging_payload( + _responses_payload([_RESPONSES_TEXT_ITEM, claude_fetch.model_dump(), client_call]), capture_content=True + ) + + assert json.loads(json.dumps(hosted_only.choices_out[0]["message"]["tool_calls"])) == [claude_tool_call] + assert hosted_only.finish_reasons == ("stop",) + assert json.loads(json.dumps(with_client_call.choices_out[0]["message"]["tool_calls"])) == [ + claude_tool_call, + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}, + ] + assert with_client_call.finish_reasons == ("tool_calls",) + + def test_responses_tool_call_only_output_has_no_content(): data = LLMCallSpanData.from_standard_logging_payload( _responses_payload([{"type": "function_call", "id": "fc_1", "name": "get_weather", "arguments": "{}"}]), diff --git a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py index 57764bf902b..611e828cf84 100644 --- a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py @@ -174,6 +174,45 @@ def test_openinference_output_tool_calls_preserve_calls_in_attributes_and_value( ] +def test_openinference_output_lists_claude_server_tool_calls_before_client_tool_calls(): + data: Final = _llm_call( + choices_out=( + { + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_paris", + "type": "function", + "function": {"name": "lookup_weather", "arguments": '{"city": "Paris"}'}, + } + ], + "provider_specific_fields": { + "server_tool_calls": [ + { + "id": "srvtoolu_01Search", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "paris weather"}'}, + } + ] + }, + }, + }, + ) + ) + attrs: Final = OpenInferenceMapper().map(data) + assert {key: value for key, value in attrs.items() if ".tool_calls." in key} == { + "llm.output_messages.0.message.tool_calls.0.tool_call.id": "srvtoolu_01Search", + "llm.output_messages.0.message.tool_calls.0.tool_call.function.name": "web_search", + "llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": '{"query": "paris weather"}', + "llm.output_messages.0.message.tool_calls.1.tool_call.id": "call_paris", + "llm.output_messages.0.message.tool_calls.1.tool_call.function.name": "lookup_weather", + "llm.output_messages.0.message.tool_calls.1.tool_call.function.arguments": '{"city": "Paris"}', + } + + def test_openinference_responses_tool_calls_are_emitted_as_output_attributes(): data: Final = LLMCallSpanData.from_standard_logging_payload( _responses_payload( diff --git a/tests/unit/integrations/test_helicone.py b/tests/unit/integrations/test_helicone.py index ce3f61a61eb..3f49d000747 100644 --- a/tests/unit/integrations/test_helicone.py +++ b/tests/unit/integrations/test_helicone.py @@ -17,7 +17,8 @@ def _claude_mapping(messages, response_obj): return logger.claude_mapping(model="gpt-5.6", messages=messages, response_obj=response_obj) -def test_claude_mapping_serializes_custom_tool_calls(monkeypatch): +@pytest.fixture() +def anthropic_prompts(monkeypatch): """ Stub the anthropic module unconditionally: the SDK may be absent (it lives in the proxy-runtime extra), and the tests/unit/llms/anthropic test package can @@ -27,6 +28,10 @@ def test_claude_mapping_serializes_custom_tool_calls(monkeypatch): stub.HUMAN_PROMPT = "\n\nHuman:" stub.AI_PROMPT = "\n\nAssistant:" monkeypatch.setitem(sys.modules, "anthropic", stub) + + +@pytest.mark.usefixtures("anthropic_prompts") +def test_claude_mapping_serializes_custom_tool_calls(): response_obj = { "id": "chatcmpl-1", "choices": [ @@ -58,6 +63,44 @@ def test_claude_mapping_serializes_custom_tool_calls(monkeypatch): assert {"type": "tool_use", "id": "call_f", "name": "read_file", "input": '{"path": "a.py"}'} in tool_use_blocks +@pytest.mark.usefixtures("anthropic_prompts") +def test_claude_mapping_puts_server_tool_calls_first_as_tool_use_blocks(): + response_obj = { + "id": "chatcmpl-1", + "choices": [ + { + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_f", + "type": "function", + "function": {"name": "read_file", "arguments": '{"path": "a.py"}'}, + }, + ], + "provider_specific_fields": { + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + ] + }, + }, + "finish_reason": "tool_calls", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 2}, + } + mapped = _claude_mapping([{"role": "user", "content": "hi"}], response_obj) + assert mapped["content"] == [ + {"type": "tool_use", "id": "srvtoolu_01", "name": "web_search", "input": '{"query": "persimmon"}'}, + {"type": "tool_use", "id": "call_f", "name": "read_file", "input": '{"path": "a.py"}'}, + ] + + @pytest.fixture() def _vcr_outcome_gate(request, vcr): install_live_call_probe(request, vcr) diff --git a/tests/unit/integrations/test_langfuse_otel.py b/tests/unit/integrations/test_langfuse_otel.py index a2803719c75..71087b4de51 100644 --- a/tests/unit/integrations/test_langfuse_otel.py +++ b/tests/unit/integrations/test_langfuse_otel.py @@ -1309,6 +1309,55 @@ class TestDerivedTraceFields: assert json.loads(str(attributes["langfuse.observation.output"])) == {"content": "hi"} + def test_server_tool_calls_lead_the_observation_output(self) -> None: + response_obj: Final = ModelResponse( + id="chatcmpl-server", + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=litellm.Message( + role="assistant", + content=None, + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "SF"}'}, + } + ], + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + ] + }, + ), + ) + ], + ) + attributes: Final = _emitted({"call_type": "acompletion", "litellm_params": {"metadata": {}}}, response_obj) + + assert json.loads(str(attributes["langfuse.observation.output"])) == [ + { + "id": "chatcmpl-server", + "name": "web_search", + "call_id": "srvtoolu_01", + "type": "function_call", + "arguments": {"query": "persimmon"}, + }, + { + "id": "chatcmpl-server", + "name": "get_weather", + "call_id": "call_1", + "type": "function_call", + "arguments": {"city": "SF"}, + }, + ] + def test_no_messages_leaves_trace_input_unset_but_emits_output(self) -> None: attributes: Final = _emitted( {"call_type": "acompletion", "litellm_params": {"metadata": {}}}, _chat_response("hi") diff --git a/tests/unit/integrations/test_lunary.py b/tests/unit/integrations/test_lunary.py index 6491f5c8b82..302f3d7e22e 100644 --- a/tests/unit/integrations/test_lunary.py +++ b/tests/unit/integrations/test_lunary.py @@ -1,6 +1,7 @@ -from litellm.integrations.lunary import parse_tool_calls +import litellm +from litellm.integrations.lunary import parse_messages, parse_tool_calls from litellm.types.utils import ( ChatCompletionMessageCustomToolCall, ChatCompletionMessageToolCall, @@ -35,3 +36,46 @@ def test_parse_tool_calls_serializes_custom_tool_calls(): def test_parse_tool_calls_none_passthrough(): assert parse_tool_calls(None) is None + + +def test_parse_messages_lists_server_tool_calls_before_client_calls(): + response = litellm.ModelResponse( + choices=[ + litellm.Choices( + index=0, + finish_reason="tool_calls", + message=litellm.Message( + role="assistant", + content=None, + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "SF"}'}, + } + ], + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + ] + }, + ), + ) + ], + ) + assert parse_messages(response.choices) == { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "type": "function", + "id": "srvtoolu_01", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + }, + {"type": "function", "id": "call_1", "function": {"name": "get_weather", "arguments": '{"city": "SF"}'}}, + ], + } diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index 5f749e951b1..0c8fdb5273e 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -372,6 +372,60 @@ def test_v1_routing_diagnostics_add_scalar_attributes_and_preserve_legacy_metada assert finished.attributes["metadata.routing_decision"] == str(decision) +def test_v1_server_tool_calls_lead_the_completion_function_call_attributes() -> None: + exporter: Final = InMemorySpanExporter() + provider: Final = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + otel: Final = OpenTelemetry(tracer_provider=provider) + response_obj: Final = litellm.ModelResponse( + choices=[ + litellm.Choices( + index=0, + finish_reason="tool_calls", + message=litellm.Message( + role="assistant", + content=None, + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "SF"}'}, + } + ], + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + ] + }, + ), + ) + ], + ) + with provider.get_tracer(__name__).start_as_current_span("inference") as span: + otel.set_attributes( + span, + { + "model": "claude-opus-5-5", + "standard_logging_object": {"id": "server-tools-call", "call_type": "acompletion", "metadata": {}}, + }, + response_obj, + ) + (finished,) = exporter.get_finished_spans() + function_call_attributes: Final = { + key: value for key, value in finished.attributes.items() if ".function_call." in key + } + assert function_call_attributes == { + f"{SpanAttributes.LLM_COMPLETIONS.value}.0.function_call.name": "web_search", + f"{SpanAttributes.LLM_COMPLETIONS.value}.0.function_call.arguments": '{"query": "persimmon"}', + f"{SpanAttributes.LLM_COMPLETIONS.value}.1.function_call.name": "get_weather", + f"{SpanAttributes.LLM_COMPLETIONS.value}.1.function_call.arguments": '{"city": "SF"}', + } + + def test_v1_pod_routing_none_matches_other_none_metadata_attributes() -> None: exporter: Final = InMemorySpanExporter() provider: Final = TracerProvider() diff --git a/tests/unit/integrations/test_shadow_eval_logger.py b/tests/unit/integrations/test_shadow_eval_logger.py index f802dbfdba9..f048821f8b7 100644 --- a/tests/unit/integrations/test_shadow_eval_logger.py +++ b/tests/unit/integrations/test_shadow_eval_logger.py @@ -1905,6 +1905,40 @@ class TestShadowPipeline: assert "Let me look that up. [tool call] Read({})" in judge_prompt + async def test_a_server_tool_call_reaches_the_judge_ahead_of_client_tool_calls(self): + router = _shadow_reply_router( + { + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "SF"}'}, + } + ], + "provider_specific_fields": { + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + ] + }, + }, + finish_reason="tool_calls", + ) + await self._judged_shadow_row(router) + + judge_prompt = next( + call.kwargs["messages"][-1]["content"] + for call in router.acompletion.call_args_list + if call.kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != SHADOW_EVAL_ROUTER_CALL_ORIGIN + ) + + shadow_text: Final = '[tool call] web_search({"query": "persimmon"}), [tool call] get_weather({"city": "SF"})' + assert f":\n{shadow_text}\n\n" in judge_prompt, judge_prompt + async def test_an_empty_shadow_reply_names_the_finish_reason_and_the_routed_model(self): """A reply that really carried no text is diagnosable only if the row says what the arm was doing when it produced none: a truncated turn and a model that answers diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 72e72714cd4..e3d58fffdda 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( converse_redacted_content, encrypted_content_from_signature, encrypted_reasoning_signature, + get_content_from_model_response, get_file_ids_from_messages, get_format_from_file_id, get_semantic_cache_prompt_from_messages, @@ -46,6 +47,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( with_prompt_cache_breakpoint, ) from litellm.types.llms.openai import ChatCompletionAudioObject, ChatCompletionFileObject, ChatCompletionImageObject +from litellm.types.utils import ChatCompletionMessageToolCall, Choices, Function, Message, ModelResponse _ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\]]{1,200}$' _ARTIFACT_DATA_ID_PATTERN: Final = r"^(?!\.\.?(?:\/|$))[A-Za-z0-9_\-.~:@+]{1,200}$" @@ -2675,3 +2677,30 @@ def test_tpm_projection_preserves_non_media_identity_and_opaque_values() -> None projected, media = split_media_for_tpm(request) assert projected is request assert media == () + + +def test_get_content_from_model_response_includes_server_tool_calls_before_client_calls() -> None: + server_call: Final = ChatCompletionMessageToolCall( + id="srvtoolu_01", type="function", function=Function(name="web_search", arguments='{"query": "persimmon"}') + ) + client_call: Final = ChatCompletionMessageToolCall( + id="call_1", type="function", function=Function(name="get_weather", arguments='{"city": "SF"}') + ) + response: Final = ModelResponse( + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=Message( + role="assistant", + content="Looking it up.", + tool_calls=[client_call.model_dump()], + provider_specific_fields={"server_tool_calls": [server_call.model_dump()]}, + ), + ) + ], + ) + + assert get_content_from_model_response(response) == ( + "Looking it up." + server_call.model_dump_json() + client_call.model_dump_json() + ) diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index 9e385c588ae..7a3c94af472 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -20,10 +20,11 @@ from unittest.mock import patch import httpx import pytest import pytest_asyncio -from pydantic import JsonValue +from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch +from pydantic import JsonValue, TypeAdapter import litellm -from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE +from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE, REDACTED_BY_LITELLM from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.redact_messages import ( @@ -37,6 +38,7 @@ from litellm.litellm_core_utils.redact_messages import ( ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.responses.main import mock_responses_api_response +from litellm.types.responses.main import build_web_search_call from litellm.types.router import BaselineRouteStamp from litellm.types.utils import ( ModelResponse, @@ -169,6 +171,114 @@ def test_streaming_redaction_redacts_delta_provider_fields_without_touching_the_ assert fields == _provider_fields() +_SECRET_URL: Final = f"https://example.com/?q={SECRET}" +_WEB_SEARCH_CALL: Final = TypeAdapter(ResponseFunctionWebSearch) + + +def _claude_fetch_call() -> ResponseFunctionWebSearch: + return build_web_search_call( + "srvtoolu_01Fetch", {"url": _SECRET_URL}, {"content": []}, status="in_progress", tool_name="web_fetch" + ) + + +def _web_call_summaries(calls: object) -> list[tuple[str, str | None, dict[str, object]]]: + items: Final = calls.values() if isinstance(calls, dict) else calls + return [ + (call.id, call.status, call.action.model_dump(exclude_none=True)) + for call in (_WEB_SEARCH_CALL.validate_python(item) for item in items) + ] + + +@pytest.mark.parametrize("shape", ["object", "dict"]) +def test_redaction_hides_claude_server_call_inputs_but_keeps_the_calls(shape: str) -> None: + response: Final = ModelResponse( + model="claude-opus-5-5", + choices=[ + { + "message": { + "role": "assistant", + "content": None, + "provider_specific_fields": { + "server_tool_calls": [ + { + "id": "srvtoolu_01Fetch", + "type": "function", + "function": {"name": "web_fetch", "arguments": json.dumps({"url": _SECRET_URL})}, + } + ], + "web_search_calls": [ + _claude_fetch_call(), + build_web_search_call( + "srvtoolu_01Search", {"query": SECRET}, {"content": []}, status="in_progress" + ), + ], + }, + } + } + ], + ) + result: Final = response if shape == "object" else response.model_dump() + + redacted: Final = perform_redaction({}, result) + + message: Final = ( + redacted.choices[0].message if shape == "object" else SimpleNamespace(**redacted["choices"][0]["message"]) + ) + fields: Final = message.provider_specific_fields + assert fields["server_tool_calls"][0]["id"] == "srvtoolu_01Fetch" + assert fields["server_tool_calls"][0]["function"] == {"name": "web_fetch", "arguments": REDACTED_BY_LITELLM} + assert _web_call_summaries(fields["web_search_calls"]) == [ + ("ws_srvtoolu_01Fetch", "in_progress", {"type": "open_page", "url": REDACTED_BY_LITELLM}), + ("ws_srvtoolu_01Search", "in_progress", {"type": "search", "query": REDACTED_BY_LITELLM}), + ] + assert SECRET not in str(redacted), redacted + assert SECRET in str(result) + + +@pytest.mark.parametrize("key", ["complete_streaming_response", "async_complete_streaming_response"]) +def test_streaming_redaction_hides_the_hosted_copy_of_a_claude_web_fetch(key: str) -> None: + response: Final = ModelResponseStream( + choices=[ + litellm.utils.StreamingChoices( + delta=litellm.utils.Delta( + provider_specific_fields={"web_search_calls": {"srvtoolu_01Fetch": _claude_fetch_call()}} + ) + ) + ] + ) + + perform_redaction({"stream": True, key: response}, None) + + assert _web_call_summaries(response.choices[0].delta.provider_specific_fields["web_search_calls"]) == [ + ("ws_srvtoolu_01Fetch", "in_progress", {"type": "open_page", "url": REDACTED_BY_LITELLM}) + ] + + +@pytest.mark.parametrize("shape", ["object", "dict"]) +def test_redaction_hides_web_search_call_actions_in_responses_api_output(shape: str) -> None: + response: Final = ResponsesAPIResponse( + id="resp_1", + created_at=1, + model="claude-opus-5-5", + object="response", + status="completed", + output=[ + build_web_search_call("srvtoolu_01Fetch", {"url": _SECRET_URL}, {"content": []}, tool_name="web_fetch"), + build_web_search_call("srvtoolu_01Search", {"query": SECRET}, {"content": []}), + ], + ) + result: Final = response if shape == "object" else response.model_dump() + + redacted: Final = perform_redaction({}, result) + + output: Final = redacted.output if shape == "object" else redacted["output"] + assert _web_call_summaries(output) == [ + ("ws_srvtoolu_01Fetch", "completed", {"type": "open_page", "url": REDACTED_BY_LITELLM}), + ("ws_srvtoolu_01Search", "completed", {"type": "search", "query": REDACTED_BY_LITELLM}), + ] + assert SECRET in str(result) + + @pytest.mark.parametrize("shape", ["object", "dict"]) def test_redaction_hides_a_compaction_summary_in_responses_api_output(shape: str) -> None: response: Final = ResponsesAPIResponse( diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py index 5d3fc523c0c..3225101cdfc 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -15,7 +15,9 @@ import litellm from litellm._uuid import uuid from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of from litellm.llms.anthropic.chat.handler import ModelResponseIterator, _envelope_verdict, make_call, make_sync_call +from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params from litellm.llms.base_llm.base_utils import response_format_schema, unwrap_response_format_envelope from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.llms.openai import ( @@ -23,7 +25,7 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallFunctionChunk, ) from litellm.types.responses.main import OutputCodeInterpreterCall -from litellm.types.utils import ChatCompletionDeltaToolCall, ModelResponseStream +from litellm.types.utils import ChatCompletionDeltaToolCall, ChatCompletionMessageToolCall, Choices, ModelResponseStream from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE @@ -1780,19 +1782,113 @@ def test_web_search_tool_result_no_extra_tool_calls(): for tc in parsed.choices[0].delta.tool_calls: tool_calls_emitted.append(tc) - # Should have exactly 2 tool calls: - # 1. From content_block_start (server_tool_use) with id and name - # 2. From content_block_delta with the actual query - assert len(tool_calls_emitted) == 2, f"Expected 2 tool calls, got {len(tool_calls_emitted)}" + assert tool_calls_emitted == [] - # First tool call should have the id and name - assert tool_calls_emitted[0]["id"] == "srvtoolu_01ABC123" - assert tool_calls_emitted[0]["function"]["name"] == "web_search" - # Second tool call should have the query arguments - assert tool_calls_emitted[1]["function"]["arguments"] == '{"query": "test"}' +def _server_tool_input_delta(fragment: str) -> dict[str, object]: + return {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": fragment}} - # The {} chunks from web_search_tool_result should NOT have been emitted as tool calls + +def _server_search_then_answer(*partial_json: str) -> tuple[dict[str, object], ...]: + return ( + { + "type": "message_start", + "message": {"id": "msg_01", "model": "claude-opus-5-5", "usage": {"input_tokens": 9}}, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "server_tool_use", "id": "srvtoolu_01Web", "name": "web_search", "input": {}}, + }, + *(_server_tool_input_delta(fragment) for fragment in partial_json), + {"type": "content_block_stop", "index": 0}, + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_01Web", "content": []}, + }, + {"type": "content_block_stop", "index": 1}, + {"type": "content_block_start", "index": 2, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": "Paris is in France."}}, + {"type": "content_block_stop", "index": 2}, + ) + + +_SERVER_SEARCH_THEN_ANSWER: Final = _server_search_then_answer('{"query": "Paris"}') +_CLIENT_TOOL_CALL: Final[tuple[dict[str, object], ...]] = ( + { + "type": "content_block_start", + "index": 3, + "content_block": {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {}}, + }, + { + "type": "content_block_delta", + "index": 3, + "delta": {"type": "input_json_delta", "partial_json": '{"city": "Paris"}'}, + }, + {"type": "content_block_stop", "index": 3}, +) + + +def _message_delta(stop_reason: str) -> dict[str, object]: + return {"type": "message_delta", "delta": {"stop_reason": stop_reason}, "usage": {"output_tokens": 20}} + + +_PARIS_SEARCH: Final = ("srvtoolu_01Web", "web_search", '{"query": "Paris"}') + + +@pytest.mark.parametrize( + ("events", "expected_tool_calls", "expected_finish_reason", "expected_server_tool_call"), + [ + pytest.param( + (*_SERVER_SEARCH_THEN_ANSWER, _message_delta("end_turn")), [], "stop", _PARIS_SEARCH, id="server_tool_only" + ), + pytest.param( + (*_SERVER_SEARCH_THEN_ANSWER, *_CLIENT_TOOL_CALL, _message_delta("tool_use")), + [("toolu_01Weather", "get_weather", '{"city": "Paris"}')], + "tool_calls", + _PARIS_SEARCH, + id="server_tool_then_client_tool", + ), + pytest.param( + (*_server_search_then_answer('{"que', 'ry": "Pa', 'ris"}'), _message_delta("end_turn")), + [], + "stop", + _PARIS_SEARCH, + id="server_tool_input_split_across_deltas", + ), + pytest.param( + (*_server_search_then_answer(), _message_delta("end_turn")), + [], + "stop", + ("srvtoolu_01Web", "web_search", "{}"), + id="server_tool_with_empty_input", + ), + ], +) +def test_streamed_server_tool_use_is_not_returned_as_a_client_tool_call( + events: tuple[dict[str, object], ...], + expected_tool_calls: list[tuple[str, str, str]], + expected_finish_reason: str, + expected_server_tool_call: tuple[str, str, str], +): + choice: Final = _assemble_stream(events).choices[0] + assert isinstance(choice, Choices) + + assert (choice.message.tool_calls is None) == (not expected_tool_calls) + assert [ + (call.id, call.function.name, call.function.arguments) + for call in choice.message.tool_calls or () + if isinstance(call, ChatCompletionMessageToolCall) + ] == expected_tool_calls + assert choice.finish_reason == expected_finish_reason + assert choice.message.content == "Paris is in France." + assert [ + (call["id"], call["function"].get("name"), call["function"].get("arguments")) + for call in anthropic_server_tool_call_params(choice.message) + ] == [expected_server_tool_call] + web_search_results: Final = message_field(choice.message.provider_specific_fields, "web_search_results") + assert [message_field(result, "tool_use_id") for result in parts_of(web_search_results)] == ["srvtoolu_01Web"] def test_current_content_block_type_tracking(): diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index cbb60caa57b..afd49667d7b 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,8 +1,9 @@ import copy import json import logging +from collections.abc import Mapping from itertools import chain -from typing import Final +from typing import Final, cast from unittest.mock import MagicMock, patch import httpx @@ -23,8 +24,13 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of from litellm.llms.anthropic.chat import ModelResponseIterator -from litellm.llms.anthropic.common_utils import process_anthropic_headers +from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, + process_anthropic_headers, + validated_object_mapping, +) from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema from litellm.llms.anthropic.messages.native.transformation import ( @@ -39,8 +45,8 @@ from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation im VertexAIAnthropicConfig, ) from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES -from litellm.types.llms.openai import ChatCompletionToolCallFunctionChunk -from litellm.types.utils import ChatCompletionToolCallChunk, ModelResponse, ServerToolUse, Usage +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallFunctionChunk +from litellm.types.utils import ChatCompletionToolCallChunk, Choices, Message, ModelResponse, ServerToolUse, Usage from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE @@ -3551,21 +3557,17 @@ def test_code_execution_tool_results_extraction(): prefix_prompt=None, ) - # Verify tool calls are present - assert transformed_response.choices[0].message.tool_calls is not None - assert len(transformed_response.choices[0].message.tool_calls) == 2 + assert transformed_response.choices[0].message.tool_calls is None - # Verify first tool call - assert transformed_response.choices[0].message.tool_calls[0].id == "srvtoolu_01ABC" - assert transformed_response.choices[0].message.tool_calls[0].function.name == "bash_code_execution" - - # Verify second tool call - assert transformed_response.choices[0].message.tool_calls[1].id == "srvtoolu_01DEF" - assert transformed_response.choices[0].message.tool_calls[1].function.name == "text_editor_code_execution" - - # Verify tool results are in provider_specific_fields provider_fields = transformed_response.choices[0].message.provider_specific_fields assert provider_fields is not None + server_tool_calls: Final = anthropic_server_tool_call_params(transformed_response.choices[0].message) + assert [(call["id"], call["function"].get("name")) for call in server_tool_calls] == [ + ("srvtoolu_01ABC", "bash_code_execution"), + ("srvtoolu_01DEF", "text_editor_code_execution"), + ] + + # Verify tool results are in provider_specific_fields assert "tool_results" in provider_fields assert provider_fields["tool_results"] is not None assert len(provider_fields["tool_results"]) == 2 @@ -3588,6 +3590,45 @@ def test_code_execution_tool_results_extraction(): assert "Done!" in transformed_response.choices[0].message.content +def test_server_and_client_tool_calls_split_by_block_type_not_id_prefix(): + completion_response: Final = { + "id": "msg_split", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5-20250929", + "content": [ + {"type": "server_tool_use", "id": "toolu_server", "name": "web_search", "input": {"query": "persimmon"}}, + {"type": "tool_use", "id": "srvtoolu_client", "name": "get_weather", "input": {"city": "Paris"}}, + ], + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + raw_response: Final = MagicMock(spec=httpx.Response) + raw_response.headers = {} + + message: Final = ( + AnthropicConfig() + .transform_parsed_response( + completion_response=completion_response, + raw_response=raw_response, + model_response=ModelResponse(), + json_mode=False, + prefix_prompt=None, + ) + .choices[0] + .message + ) + + assert [ + (call["id"], call["function"].get("name"), call["function"].get("arguments")) + for call in anthropic_server_tool_call_params(message) + ] == [("toolu_server", "web_search", '{"query": "persimmon"}')] + assert [ + (tool_call.id, tool_call.function.name, tool_call.function.arguments) for tool_call in message.tool_calls or () + ] == [("srvtoolu_client", "get_weather", '{"city": "Paris"}')] + + def test_code_execution_tool_results_in_hidden_params(): """ Test that tool_results reaches _hidden_params so the Responses API adapter @@ -3675,7 +3716,7 @@ def _tool_search_turn(tool_references: list[str], last_block: dict[str, object]) ] -def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str): +def _parsed_choice(content: list[dict[str, object]], stop_reason: str) -> Choices: anthropic_response: Final = { "id": "msg_01XYZ", "type": "message", @@ -3690,7 +3731,7 @@ def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str raw_response.json.return_value = anthropic_response raw_response.status_code = 200 raw_response.headers = {} - return ( + choice: Final = ( AnthropicConfig() .transform_parsed_response( completion_response=anthropic_response, @@ -3700,14 +3741,19 @@ def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str prefix_prompt=None, ) .choices[0] - .message ) + assert isinstance(choice, Choices) + return choice + + +def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str) -> Message: + return _parsed_choice(content, stop_reason).message _WEATHER_CALL: Final = {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}} -def _web_search_turn(last_block: dict[str, object]) -> list[dict[str, object]]: +def _web_search_turn(last_block: Mapping[str, object]) -> list[dict[str, object]]: return [ {"type": "server_tool_use", "id": "srvtoolu_01Web", "name": "web_search", "input": {"query": "Paris facts"}}, { @@ -3723,7 +3769,7 @@ def _web_search_turn(last_block: dict[str, object]) -> list[dict[str, object]]: } ], }, - last_block, + dict(last_block), ] @@ -3770,8 +3816,114 @@ def test_server_tool_result_round_trips_right_after_its_call( assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"] -def _block_ids(message: dict[str, object], block_type: str, id_key: str) -> list[object]: - return sorted(block[id_key] for block in message["content"] if block["type"] == block_type) +_FETCH_STILL_WAITING_ON_ITS_RESULT: Final = { + "type": "server_tool_use", + "id": "srvtoolu_01Fetch", + "name": "web_fetch", + "input": {"url": "https://example.com/"}, +} + + +@pytest.mark.parametrize("modify_params", [False, True]) +def test_server_call_still_waiting_on_its_result_is_replayed_as_a_server_call( + modify_params: bool, monkeypatch: pytest.MonkeyPatch +): + content: Final = [_FETCH_STILL_WAITING_ON_ITS_RESULT, _WEATHER_CALL] + message: Final = _parsed_assistant_message(content, "tool_use") + monkeypatch.setattr(litellm, "modify_params", modify_params) + + replayed: Final = _replayed( + {"role": "user", "content": "Fetch example.com, then tell me the weather in Paris"}, + message.model_dump(), + {"role": "tool", "tool_call_id": "toolu_01Weather", "content": "18C"}, + ) + + assert replayed[1]["content"] == content + assert [turn["role"] for turn in replayed] == ["user", "assistant", "user"] + assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"] + + +_PARIS_TEXT: Final = "Paris is the capital of France." +_PARIS_ANSWER: Final[Mapping[str, str]] = {"type": "text", "text": _PARIS_TEXT} + + +@pytest.mark.parametrize( + ("content", "stop_reason", "expected_tool_call_ids", "expected_finish_reason"), + [ + pytest.param(_web_search_turn(_PARIS_ANSWER), "end_turn", [], "stop", id="web_search_then_answer"), + pytest.param( + _web_search_turn(_WEATHER_CALL), "tool_use", ["toolu_01Weather"], "tool_calls", id="web_search_then_tool" + ), + pytest.param([_WEATHER_CALL], "tool_use", ["toolu_01Weather"], "tool_calls", id="client_tool_only"), + ], +) +def test_server_tool_use_is_not_returned_as_a_client_tool_call( + content: list[dict[str, object]], + stop_reason: str, + expected_tool_call_ids: list[str], + expected_finish_reason: str, +): + choice: Final = _parsed_choice(content, stop_reason) + + assert (choice.message.tool_calls is None) == (not expected_tool_call_ids) + assert [call.id for call in choice.message.tool_calls or ()] == expected_tool_call_ids + assert choice.finish_reason == expected_finish_reason + assert [ + (call["id"], call["function"].get("name"), call["function"].get("arguments")) + for call in anthropic_server_tool_call_params(choice.message) + ] == [ + (block["id"], block["name"], json.dumps(block["input"])) + for block in content + if block["type"] == "server_tool_use" + ] + if not expected_tool_call_ids: + assert choice.message.content == _PARIS_TEXT + + +def _replayed(*messages: object): + return anthropic_messages_pt( + messages=cast(list[AllMessageValues], list(messages)), # cast-ok: replays history the way clients send it back + model="claude-opus-5-5", + llm_provider="anthropic", + ) + + +def test_web_search_answer_round_trips_without_any_client_tool_call(): + message: Final = _parsed_assistant_message(_web_search_turn(_PARIS_ANSWER), "end_turn") + + replayed: Final = _replayed( + {"role": "user", "content": "Tell me about Paris"}, + message.model_dump(), + {"role": "user", "content": "And Lyon?"}, + ) + + assert _block_ids(replayed[1], "server_tool_use", "id") == ["srvtoolu_01Web"] + assert _block_ids(replayed[1], "web_search_tool_result", "tool_use_id") == ["srvtoolu_01Web"] + assert [ + message_field(block, "text") + for block in parts_of(replayed[1]["content"]) + if message_field(block, "type") == "text" + ] == [_PARIS_TEXT] + + +def _block_ids(message: Mapping[str, object], block_type: str, id_key: str) -> list[str]: + return sorted( + block_id + for block in parts_of(message["content"]) + if message_field(block, "type") == block_type and isinstance(block_id := message_field(block, id_key), str) + ) + + +def _as_returned_before_server_tool_calls_moved(message: Message) -> Mapping[str, object]: + dumped: Final[Mapping[str, object]] = message.model_dump() + provider_fields: Final = validated_object_mapping(dumped.get("provider_specific_fields")) or {} + return { + **dumped, + "tool_calls": [*anthropic_server_tool_call_params(message), *parts_of(dumped.get("tool_calls"))], + "provider_specific_fields": { + key: value for key, value in provider_fields.items() if key != "server_tool_calls" + }, + } _SEARCH_THAT_FOUND_NOTHING: Final = _tool_search_turn( @@ -3806,21 +3958,33 @@ def test_replayed_search_that_found_nothing_answers_every_tool_use_and_ends_on_a server_tool_use_ids: list[str], monkeypatch: pytest.MonkeyPatch, ): - from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt - message: Final = _parsed_assistant_message(_SEARCH_THAT_FOUND_NOTHING, "end_turn") monkeypatch.setattr(litellm, "modify_params", modify_params) - replayed: Final = anthropic_messages_pt( - messages=[{"role": "user", "content": "Find a tool and use it"}, message.model_dump(), *follow_ups], - model="claude-opus-5-5", - llm_provider="anthropic", + replayed: Final = _replayed( + {"role": "user", "content": "Find a tool and use it"}, + _as_returned_before_server_tool_calls_moved(message), + *follow_ups, ) assert [turn["role"] for turn in replayed] == ["user", "assistant", "user"] assert _block_ids(replayed[2], "tool_result", "tool_use_id") == _block_ids(replayed[1], "tool_use", "id") assert _block_ids(replayed[1], "server_tool_use", "id") == server_tool_use_ids +def test_replayed_search_that_found_nothing_keeps_its_result_when_the_call_never_reached_the_client( + monkeypatch: pytest.MonkeyPatch, +): + message: Final = _parsed_assistant_message(_SEARCH_THAT_FOUND_NOTHING, "end_turn") + monkeypatch.setattr(litellm, "modify_params", True) + + replayed: Final = _replayed({"role": "user", "content": "Find a tool and use it"}, message.model_dump()) + + assert [turn["role"] for turn in replayed] == ["user", "assistant"] + assert _block_ids(replayed[1], "server_tool_use", "id") == ["srvtoolu_01Search"] + assert _block_ids(replayed[1], "tool_search_tool_result", "tool_use_id") == ["srvtoolu_01Search"] + assert _block_ids(replayed[1], "tool_use", "id") == [] + + _CODE_EXECUTION_BLOCKS: Final = [ { "type": "server_tool_use", diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index e0882653324..3c4648c00f0 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -81,6 +81,59 @@ def test_native_compaction_response_roundtrip(text: str | None, count: int, expe assert result["stop_reason"] == expected_stop +_SERVER_AND_CLIENT_TOOL_TURN: Final[tuple[dict[str, object], ...]] = ( + { + "type": "server_tool_use", + "id": "srvtoolu_01Fetch", + "name": "web_fetch", + "input": {"url": "https://example.com/"}, + }, + { + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_01Fetch", + "content": {"type": "web_fetch_result", "url": "https://example.com/"}, + }, + {"type": "server_tool_use", "id": "srvtoolu_02Search", "name": "web_search", "input": {"query": "Paris"}}, + {"type": "text", "text": "The page is titled Example."}, + {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}}, +) + + +def test_claude_server_tool_blocks_survive_the_round_trip_through_chat_completions() -> None: + from unittest.mock import MagicMock + + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + anthropic_response: Final = { + "id": "msg_01Fetch", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": list(_SERVER_AND_CLIENT_TOOL_TURN), + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 100, "output_tokens": 50}, + } + raw_response: Final = MagicMock(spec=httpx.Response) + raw_response.json.return_value = anthropic_response + raw_response.status_code = 200 + raw_response.headers = {} + chat_response: Final = AnthropicConfig().transform_parsed_response( + completion_response=anthropic_response, + raw_response=raw_response, + model_response=ModelResponse(), + json_mode=False, + prefix_prompt=None, + ) + + result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(chat_response) + + assert [(block["type"], block.get("id") or block.get("tool_use_id")) for block in result["content"]] == [ + (block["type"], block.get("id") or block.get("tool_use_id")) for block in _SERVER_AND_CLIENT_TOOL_TURN + ] + assert result["stop_reason"] == "tool_use" + + def test_translate_chat_refusal_to_anthropic_response(): response = ModelResponse( id="chatcmpl-refusal", diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 5d71cbe9afc..365bb0ebfef 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -7,7 +7,7 @@ with guardrail transformations, including tool calls. import json from collections.abc import Mapping -from typing import Any, Literal, Optional +from typing import Any, Final, Literal, Optional import pytest @@ -17,6 +17,7 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import Streami from litellm.llms.openai.chat.guardrail_translation.handler import ( OpenAIChatCompletionsHandler, ) +from litellm.types.responses.main import build_web_search_call from litellm.types.utils import ( ChatCompletionMessageToolCall, Choices, @@ -144,6 +145,53 @@ class MockMisalignedToolCallGuardrail(CustomGuardrail): ) +def _server_tool_call(call_id: str, name: str, arguments: Mapping[str, object]) -> dict[str, object]: + return {"id": call_id, "type": "function", "function": {"name": name, "arguments": json.dumps(arguments)}} + + +def _tool_call_ids_names_and_arguments(tool_calls) -> list[tuple[str, str, str]]: + return [ + (tool_call["id"], tool_call["function"]["name"], tool_call["function"]["arguments"]) + for tool_call in tool_calls + ] + + +def _hosted_web_call_inputs(web_search_calls) -> list[tuple[str, str, Optional[str]]]: + return [ + (call.id, call.status, call.action.query if call.action.type == "search" else call.action.url) + for call in web_search_calls + ] + + +_SEARCH_INPUT: Final = {"query": "jane@example.com"} +_FETCH_INPUT: Final = {"url": "https://example.com/?email=jane@example.com"} + + +class QueryRedactingGuardrail(CustomGuardrail): + """Returns copied tool calls with any ``query`` or ``url`` argument redacted, leaving other calls as they came.""" + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + def redacted(tool_call): + arguments = json.loads(tool_call["function"]["arguments"]) + redacted_arguments = { + key: "[REDACTED]" if key in ("query", "url") else value for key, value in arguments.items() + } + if redacted_arguments == arguments: + return tool_call + return {**tool_call, "function": {**tool_call["function"], "arguments": json.dumps(redacted_arguments)}} + + return GenericGuardrailAPIInputs( + texts=inputs.get("texts", []), + tool_calls=[redacted(tool_call) for tool_call in inputs.get("tool_calls", [])], + ) + + class TestOpenAIChatCompletionsHandlerToolsInput: """Test input processing with tools (function definitions)""" @@ -906,6 +954,132 @@ class TestOpenAIChatCompletionsHandlerToolCallsOutput: assert tool_calls[0].function.arguments == first_args assert tool_calls[1].function.arguments == second_args + @pytest.mark.asyncio + async def test_output_guardrail_sees_a_response_that_only_carries_server_tool_calls(self): + handler = OpenAIChatCompletionsHandler() + guardrail = TestOpenAIChatCompletionsHandlerStreamingOutput._recording_guardrail() + response = ModelResponse( + model="claude-sonnet-4-5", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message( + content=None, + role="assistant", + provider_specific_fields={ + "server_tool_calls": [_server_tool_call("srvtoolu_01Search", "web_search", {"query": "x"})] + }, + ), + ) + ], + ) + + await handler.process_output_response(response, guardrail) + + assert _tool_call_ids_names_and_arguments(guardrail.seen_inputs[-1]["tool_calls"]) == [ + ("srvtoolu_01Search", "web_search", '{"query": "x"}') + ] + + @pytest.mark.asyncio + async def test_output_guardrail_rewrite_of_a_server_tool_call_lands_in_server_tool_calls_only(self): + handler = OpenAIChatCompletionsHandler() + weather_args = json.dumps({"city": "Paris", "query": "forecast"}) + response = ModelResponse( + model="claude-sonnet-4-5", + choices=[ + Choices( + finish_reason="tool_calls", + index=0, + message=Message( + content="Looking it up", + role="assistant", + tool_calls=[ + ChatCompletionMessageToolCall( + id="call_1", + type="function", + function=Function(name="get_weather", arguments=json.dumps({"city": "Paris"})), + ) + ], + provider_specific_fields={ + "server_tool_calls": [ + _server_tool_call("srvtoolu_01Search", "web_search", {"query": "jane@example.com"}) + ] + }, + ), + ), + Choices( + finish_reason="tool_calls", + index=1, + message=Message( + content=None, + role="assistant", + tool_calls=[ + ChatCompletionMessageToolCall( + id="call_2", + type="function", + function=Function(name="get_weather", arguments=weather_args), + ) + ], + ), + ), + ], + ) + + await handler.process_output_response(response, QueryRedactingGuardrail(guardrail_name="redact")) + + first, second = response.choices + assert _tool_call_ids_names_and_arguments(first.message.provider_specific_fields["server_tool_calls"]) == [ + ("srvtoolu_01Search", "web_search", '{"query": "[REDACTED]"}') + ] + assert [(call.id, call.function.arguments) for call in first.message.tool_calls] == [ + ("call_1", '{"city": "Paris"}') + ] + assert [(call.id, call.function.arguments) for call in second.message.tool_calls] == [ + ("call_2", '{"city": "Paris", "query": "[REDACTED]"}') + ] + assert not (second.message.provider_specific_fields or {}).get("server_tool_calls") + + @pytest.mark.asyncio + async def test_output_guardrail_rewrite_of_a_server_web_call_reaches_its_hosted_copy(self): + handler = OpenAIChatCompletionsHandler() + response = ModelResponse( + model="claude-sonnet-4-5", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message( + content=None, + role="assistant", + provider_specific_fields={ + "server_tool_calls": [ + _server_tool_call("srvtoolu_01Search", "web_search", _SEARCH_INPUT), + _server_tool_call("srvtoolu_01Fetch", "web_fetch", _FETCH_INPUT), + ], + "web_search_calls": [ + build_web_search_call("srvtoolu_01Search", _SEARCH_INPUT, {"content": []}), + build_web_search_call( + "srvtoolu_01Fetch", + _FETCH_INPUT, + {"content": []}, + status="in_progress", + tool_name="web_fetch", + ), + ], + }, + ), + ) + ], + ) + + await handler.process_output_response(response, QueryRedactingGuardrail(guardrail_name="redact")) + + assert _hosted_web_call_inputs(response.choices[0].message.provider_specific_fields["web_search_calls"]) == [ + ("ws_srvtoolu_01Search", "completed", "[REDACTED]"), + ("ws_srvtoolu_01Fetch", "in_progress", "[REDACTED]"), + ] + class MockPassThroughGuardrail(CustomGuardrail): """Mock guardrail that passes through without blocking - for testing streaming fallback behavior""" @@ -1462,6 +1636,130 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert chunks[0].choices[0].delta.content == "hello [MASKED]" assert chunks[1].choices[0].delta.content in (None, "") + @staticmethod + def _ended_server_tool_call_stream_chunks() -> list: + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + def chunk(delta: Delta, finish_reason: Optional[str] = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-123", + created=1234567890, + model="claude-sonnet-4-5", + object="chat.completion.chunk", + choices=[StreamingChoices(index=0, delta=delta, finish_reason=finish_reason)], + ) + + search = {**_server_tool_call("srvtoolu_01Search", "web_search", {"query": "jane@example.com"}), "index": 0} + return [ + chunk(Delta(content="Searching")), + chunk(Delta(content=None, provider_specific_fields={"server_tool_calls": [search]})), + chunk(Delta(content=None), finish_reason="stop"), + ] + + @staticmethod + def _ended_server_fetch_stream_chunks() -> list: + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + def chunk(delta: Delta, finish_reason: Optional[str] = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-123", + created=1234567890, + model="claude-sonnet-4-5", + object="chat.completion.chunk", + choices=[StreamingChoices(index=0, delta=delta, finish_reason=finish_reason)], + ) + + def hosted_fetch(tool_input: dict, result: dict, status: Optional[str]) -> dict: + return { + "srvtoolu_01Fetch": build_web_search_call( + "srvtoolu_01Fetch", tool_input, result, status=status, tool_name="web_fetch" + ) + } + + fetch = {**_server_tool_call("srvtoolu_01Fetch", "web_fetch", _FETCH_INPUT), "index": 0} + return [ + chunk( + Delta( + content=None, + provider_specific_fields={"web_search_calls": hosted_fetch({}, {"content": []}, "in_progress")}, + ) + ), + chunk( + Delta( + content=None, + provider_specific_fields={ + "server_tool_calls": [fetch], + "web_search_calls": hosted_fetch(_FETCH_INPUT, {"content": []}, "in_progress"), + }, + ) + ), + chunk( + Delta( + content=None, + provider_specific_fields={ + "web_search_calls": hosted_fetch(_FETCH_INPUT, {"content": {"type": "web_fetch_result"}}, None) + }, + ) + ), + chunk(Delta(content="Fetched"), finish_reason="stop"), + ] + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_reach_every_hosted_copy_of_a_server_web_call(self): + handler = OpenAIChatCompletionsHandler() + chunks = self._ended_server_fetch_stream_chunks() + + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=QueryRedactingGuardrail(guardrail_name="redact"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert [ + _hosted_web_call_inputs(chunk.choices[0].delta.provider_specific_fields["web_search_calls"].values()) + for chunk in chunks[:3] + ] == [ + [("ws_srvtoolu_01Fetch", "in_progress", "[REDACTED]")], + [("ws_srvtoolu_01Fetch", "in_progress", "[REDACTED]")], + [("ws_srvtoolu_01Fetch", "completed", "[REDACTED]")], + ] + + @pytest.mark.asyncio + async def test_ended_stream_guardrail_sees_streamed_server_tool_calls(self): + handler = OpenAIChatCompletionsHandler() + guardrail = self._recording_guardrail() + + await handler.process_output_streaming_response( + responses_so_far=self._ended_server_tool_call_stream_chunks(), + guardrail_to_apply=guardrail, + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert _tool_call_ids_names_and_arguments(guardrail.seen_inputs[-1]["tool_calls"]) == [ + ("srvtoolu_01Search", "web_search", '{"query": "jane@example.com"}') + ] + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_writes_server_tool_call_arguments_back_into_chunks(self): + handler = OpenAIChatCompletionsHandler() + chunks = self._ended_server_tool_call_stream_chunks() + + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=QueryRedactingGuardrail(guardrail_name="redact"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + delta = chunks[1].choices[0].delta + assert _tool_call_ids_names_and_arguments(delta.provider_specific_fields["server_tool_calls"]) == [ + ("srvtoolu_01Search", "web_search", '{"query": "[REDACTED]"}') + ] + assert delta.tool_calls is None + assert chunks[0].choices[0].delta.content == "Searching" + class TestUndecoratedGuardrailIsRecorded: """LIT-5983 regression: the handler calls apply_guardrail bare, so a custom guardrail @@ -2323,6 +2621,29 @@ class TestStreamingScanKey: assert len(ended_key.tool_calls) == 1 and "run_shell" in ended_key.tool_calls[0] assert ended_key.tool_calls_in_flight is False + def test_server_tool_calls_hold_the_window_and_enter_the_key_once_the_stream_has_ended(self): + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + handler = OpenAIChatCompletionsHandler() + server_chunk = ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta( + content=None, + provider_specific_fields={ + "server_tool_calls": [_server_tool_call("srvtoolu_01Search", "web_search", {"query": "x"})] + }, + ), + ) + ] + ) + open_key = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk]) + ended_key = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk, self._chunk(None, "stop")]) + assert open_key.tool_calls_in_flight is True + assert len(ended_key.tool_calls) == 1 and "web_search" in ended_key.tool_calls[0] + assert ended_key != open_key + def test_text_after_the_first_choice_finishes_still_changes_the_key(self): handler = OpenAIChatCompletionsHandler() first_done = [self._chunk("a", index=0), self._chunk("b", finish_reason="stop", index=0)] diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index d732d922375..1d5d7e4ed5c 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -20,7 +20,9 @@ import pytest from fastapi import HTTPException from pydantic import BaseModel +from openai.types.responses.response_function_web_search import ActionSearch from openai.types.responses import ( + ResponseFunctionWebSearch, ResponseCustomToolCall, ResponseCustomToolCallInputDeltaEvent, ResponseCustomToolCallInputDoneEvent, @@ -48,6 +50,7 @@ from litellm.types.llms.openai import ResponsesAPIResponse from litellm.responses.litellm_completion_transformation.tool_search import build_tool_search_call from litellm.types.responses.main import ( CustomToolCallOutputItem, + build_web_search_call, GenericResponseOutputItem, OutputText, OutputToolSearchCall, @@ -237,6 +240,30 @@ CUSTOM_TOOL_CALL_ITEM = { } +def _claude_web_calls_and_a_native_search() -> list[ResponseFunctionWebSearch]: + return [ + build_web_search_call("srvtoolu_1", {"query": "persimmon"}, None, "completed"), + build_web_search_call("srvtoolu_2", {"url": "https://persimmon.example"}, None, "in_progress", "web_fetch"), + ResponseFunctionWebSearch( + id="ws_native", + type="web_search_call", + status="completed", + action=ActionSearch(type="search", query="persimmon"), + ), + ] + + +def _web_call_actions(items: list[object]) -> list[dict[str, object]]: + return [ResponseFunctionWebSearch.model_validate(item, from_attributes=True).action.model_dump() for item in items] + + +_MASKED_WEB_CALL_ACTIONS: Final = [ + {"type": "search", "query": "[MASKED]", "queries": ["[MASKED]"], "sources": []}, + {"type": "open_page", "url": "https://[MASKED].example"}, + {"type": "search", "query": "persimmon", "queries": None, "sources": None}, +] + + class TestOpenAIResponsesHandlerDiscovery: """Test that the handler is properly discovered by the guardrail system""" @@ -1810,6 +1837,27 @@ class TestOpenAIResponsesHandlerToolCallExtraction: assert search_call.call_id == "srvtoolu_1" assert "name" not in search_call.model_dump() + @pytest.mark.asyncio + async def test_process_output_response_scans_and_rewrites_claude_web_calls_but_not_native_search(self): + guardrail = RecordingPersimmonMaskingGuardrail(guardrail_name="mask") + response = ResponsesAPIResponse( + id="resp_1", + created_at=1, + model="claude-sonnet-5-5", + object="response", + status="completed", + output=_claude_web_calls_and_a_native_search(), + ) + + result = await OpenAIResponsesHandler().process_output_response(response, guardrail) + + assert guardrail.seen_tool_calls == [ + ("web_search", '{"query": "persimmon"}'), + ("web_fetch", '{"url": "https://persimmon.example"}'), + ] + assert [item.type for item in result.output] == ["web_search_call"] * 3 + assert _web_call_actions(result.output) == _MASKED_WEB_CALL_ACTIONS + @pytest.mark.asyncio async def test_process_output_response_never_returns_original_search_arguments_after_a_non_json_rewrite(self): response = ResponsesAPIResponse( @@ -2745,6 +2793,117 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: assert completed.response.output[0].arguments == {"query": "[MASKED]"} assert "name" not in done.item.model_dump() + @pytest.mark.asyncio + async def test_deliver_ended_stream_scans_and_rewrites_claude_web_calls_but_not_native_search(self): + from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject + from litellm.types.llms.openai import ( + OutputItemAddedEvent, + OutputItemDoneEvent, + ResponseCompletedEvent, + ResponsesAPIStreamEvents, + ) + + items = _claude_web_calls_and_a_native_search() + added = [ + OutputItemAddedEvent( + type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, + output_index=index, + item=BaseLiteLLMOpenAIResponseObject(**{**item.model_dump(), "action": None}), + ) + for index, item in enumerate(items) + ] + done = [ + OutputItemDoneEvent( + type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, + output_index=index, + sequence_number=index + 3, + item=BaseLiteLLMOpenAIResponseObject(**item.model_dump()), + ) + for index, item in enumerate(items) + ] + completed = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=ResponsesAPIResponse( + id="resp_1", + created_at=1, + model="claude-sonnet-5-5", + object="response", + status="completed", + output=_claude_web_calls_and_a_native_search(), + ), + ) + guardrail = RecordingPersimmonMaskingGuardrail(guardrail_name="mask") + + await OpenAIResponsesHandler().process_output_streaming_response( + responses_so_far=[*added, *done, completed], + guardrail_to_apply=guardrail, + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert sorted(guardrail.seen_tool_calls) == [ + ("web_fetch", '{"url": "https://persimmon.example"}'), + ("web_search", '{"query": "persimmon"}'), + ] + assert [event.item.model_dump()["action"] for event in added] == [None, None, None] + assert _web_call_actions([event.item for event in done]) == _MASKED_WEB_CALL_ACTIONS + assert _web_call_actions(completed.response.output) == _MASKED_WEB_CALL_ACTIONS + + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_dict_shaped_claude_web_calls_but_not_native_search(self): + items = [item.model_dump() for item in _claude_web_calls_and_a_native_search()] + done = [ + {"type": "response.output_item.done", "output_index": index, "item": dict(item)} + for index, item in enumerate(items) + ] + completed = { + "type": "response.completed", + "response": { + "id": "resp_1", + "model": "claude-sonnet-5-5", + "status": "completed", + "output": [dict(item) for item in items], + }, + } + guardrail = RecordingPersimmonMaskingGuardrail(guardrail_name="mask") + + await OpenAIResponsesHandler().process_output_streaming_response( + responses_so_far=[*done, completed], + guardrail_to_apply=guardrail, + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert sorted(guardrail.seen_tool_calls) == [ + ("web_fetch", '{"url": "https://persimmon.example"}'), + ("web_search", '{"query": "persimmon"}'), + ] + assert _web_call_actions([event["item"] for event in done]) == _MASKED_WEB_CALL_ACTIONS + assert _web_call_actions(completed["response"]["output"]) == _MASKED_WEB_CALL_ACTIONS + + @pytest.mark.asyncio + async def test_streaming_output_item_done_reports_a_claude_web_search_call(self): + from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject + from litellm.types.llms.openai import OutputItemDoneEvent, ResponsesAPIStreamEvents + + done = OutputItemDoneEvent( + type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, + output_index=0, + sequence_number=1, + item=BaseLiteLLMOpenAIResponseObject( + **build_web_search_call("srvtoolu_1", {"query": "persimmon"}, None, "completed").model_dump() + ), + ) + guardrail = RecordingPersimmonMaskingGuardrail(guardrail_name="mask") + + await OpenAIResponsesHandler().process_output_streaming_response( + responses_so_far=[done], + guardrail_to_apply=guardrail, + litellm_logging_obj=None, + ) + + assert guardrail.seen_tool_calls == [("web_search", '{"query": "persimmon"}')] + @pytest.mark.asyncio async def test_deliver_ended_stream_rewrites_a_native_tool_search_call_that_has_no_call_id(self): def search_call(arguments: dict, status: str) -> dict: diff --git a/tests/unit/proxy/db/test_spend_log_tool_index.py b/tests/unit/proxy/db/test_spend_log_tool_index.py index af3268a7cc9..a3f400bbc15 100644 --- a/tests/unit/proxy/db/test_spend_log_tool_index.py +++ b/tests/unit/proxy/db/test_spend_log_tool_index.py @@ -23,6 +23,7 @@ from litellm.proxy.db.spend_log_tool_index import ( ) from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.utils import ModelResponse from tests.unit.proxy.db.fake_prisma_engine import engine_call @@ -204,6 +205,36 @@ class TestResponseToolCallNames: def test_blank_names_are_dropped(self): assert response_tool_call_names(_response_with_tool_calls(" ", "real_tool")) == ("real_tool",) + @pytest.mark.parametrize("as_object", [False, True], ids=["dict", "model_response"]) + def test_server_tool_calls_of_every_choice_are_named_before_client_tool_calls(self, as_object: bool): + def tool_call(call_id: str, name: str) -> dict[str, object]: + return {"id": call_id, "type": "function", "function": {"name": name, "arguments": "{}"}} + + response: dict[str, object] = { + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [tool_call("call_1", "get_weather")], + "provider_specific_fields": {"server_tool_calls": [tool_call("srvtoolu_01", "web_search")]}, + }, + }, + { + "index": 1, + "message": { + "role": "assistant", + "content": "done", + "provider_specific_fields": {"server_tool_calls": [tool_call("srvtoolu_02", "web_fetch")]}, + }, + }, + ] + } + completion_response = ModelResponse(**response) if as_object else response + + assert response_tool_call_names(completion_response) == ("web_search", "web_fetch", "get_weather") + def test_responses_api_output_function_calls(self): # Regression: /v1/responses carries invocations in output[] items of # type function_call, not in choices; they must reach the rollup. diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_cato_networks.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_cato_networks.py index 349030c6c75..7741973c15b 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_cato_networks.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_cato_networks.py @@ -2237,6 +2237,83 @@ async def test_post_call_success_hook_redacts_both_content_and_tool_arguments(): assert message.tool_calls[0].function.arguments == '{"to": "[NAME_1]"}' +@pytest.mark.asyncio +async def test_post_call_success_hook_inspects_and_redacts_server_tool_call_arguments(): + guard = _make_guardrail() + request_data = {"messages": [{"role": "user", "content": "look up persimmon"}]} + + def side_effect(url, *args, **kwargs): + last = kwargs["json"]["messages"][-1]["content"] + return _make_response( + { + "analysis_result": {"policy_drill_down": {"PII": {}}}, + "required_action": {"action_type": "anonymize_action", "policy_name": "PII"}, + "redacted_chat": { + "all_redacted_messages": [ + {"role": "user", "content": "look up persimmon"}, + {"role": "assistant", "content": last.replace("persimmon", "[FRUIT_1]")}, + ] + }, + } + ) + + llm_response = ModelResponse( + choices=[ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": None, + "role": "assistant", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + } + ], + "provider_specific_fields": { + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + ] + }, + }, + } + ] + ) + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + side_effect=side_effect, + ) as mock_post: + result = await guard.async_post_call_success_hook( + data=request_data, + response=llm_response, + user_api_key_dict=UserAPIKeyAuth(), + ) + + inspected = [call.kwargs["json"]["messages"][-1] for call in mock_post.call_args_list] + assert inspected == [ + {"role": "assistant", "content": '{"query": "persimmon"}'}, + {"role": "assistant", "content": '{"city": "Paris"}'}, + ] + message = result.choices[0].message + assert message.provider_specific_fields == { + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "[FRUIT_1]"}'}, + } + ] + } + assert message.content is None + assert message.tool_calls[0].function.arguments == '{"city": "Paris"}' + + def _make_responses_api_response(output: list) -> ResponsesAPIResponse: return ResponsesAPIResponse(id="resp-1", created_at=0, output=output) diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py index 5ce78fd05d1..740c3c90758 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py @@ -404,6 +404,33 @@ class TestCiscoAIDefenseChatMode: ) +_CLAUDE_HOSTED_WEB_ACTIONS = ( + {"type": "search", "query": "ssn 123-45-6789"}, + {"type": "open_page", "url": "https://example.com/?ssn=123-45-6789"}, +) + + +def _claude_hosted_web_call_response(action: dict): + from openai.types.responses import ResponseFunctionWebSearch + + from litellm.types.llms.openai import ResponsesAPIResponse + + return ResponsesAPIResponse( + id="resp_1", + created_at=0, + output=[ + ResponseFunctionWebSearch( + id="ws_srvtoolu_01", type="web_search_call", status="completed", action=action + ) + ], + parallel_tool_calls=False, + tool_choice=None, + tools=None, + top_p=None, + usage=None, + ) + + class TestCiscoAIDefenseResponsesAPIOutput: @staticmethod @@ -505,6 +532,23 @@ class TestCiscoAIDefenseResponsesAPIOutput: joined = " ".join(m.get("content", "") for m in (sent.get("messages") or [])) assert "4111-1111-1111-1111" in joined + @pytest.mark.parametrize("action", _CLAUDE_HOSTED_WEB_ACTIONS) + @pytest.mark.asyncio + async def test_post_call_scans_claude_hosted_web_call(self, action): + g = _make_guardrail(event_hook="post_call") + post_mock = AsyncMock(return_value=_safe_response()) + + with _patch_inspection_post(g, post_mock): + await g.async_post_call_success_hook( + data={"input": [{"role": "user", "content": "look it up"}]}, + user_api_key_dict=UserAPIKeyAuth(), + response=_claude_hosted_web_call_response(action), + ) + + sent = post_mock.call_args.kwargs["json"] + joined = " ".join(m.get("content", "") for m in (sent.get("messages") or [])) + assert "123-45-6789" in joined + @pytest.mark.asyncio async def test_post_call_responses_api_violation_is_blocked(self): g = _make_guardrail(event_hook="post_call") @@ -573,6 +617,24 @@ class TestCiscoAIDefenseResponsesAPIOutputRedaction: ) + @pytest.mark.parametrize("action", _CLAUDE_HOSTED_WEB_ACTIONS) + @pytest.mark.asyncio + async def test_redact_clears_claude_hosted_web_call(self, action): + g = _make_guardrail(event_hook="post_call", on_flagged_action="monitor") + cisco_resp = _redact_response(sanitized_text="[REDACTED]") + + with _patch_inspection_post(g, AsyncMock(return_value=cisco_resp)): + result = await g.async_post_call_success_hook( + data={"input": [{"role": "user", "content": "look it up"}]}, + user_api_key_dict=UserAPIKeyAuth(), + response=_claude_hosted_web_call_response(action), + ) + + call = result.output[0] + assert (call.id, call.action.type) == ("ws_srvtoolu_01", action["type"]) + assert "123-45-6789" not in call.action.model_dump_json() + + class TestCiscoAIDefenseResponsesAPIInputRedaction: @pytest.mark.parametrize( @@ -1777,6 +1839,78 @@ class TestCiscoAIDefenseToolCallBypass: f"included in the scanned text. Sent: {sent!r}" ) + @staticmethod + def _server_tool_call_response() -> ModelResponse: + from litellm.types.utils import ChatCompletionMessageToolCall, Function + + return ModelResponse( + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=[ + ChatCompletionMessageToolCall( + id="call_1", + type="function", + function=Function(name="get_weather", arguments='{"city": "Paris"}'), + ) + ], + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + ] + }, + ), + ) + ] + ) + + @pytest.mark.asyncio + async def test_post_call_scan_sends_server_tool_call_arguments_to_cisco(self): + g = _make_guardrail(event_hook="post_call") + post_mock = AsyncMock(return_value=_safe_response()) + + with _patch_inspection_post(g, post_mock): + await g.async_post_call_success_hook( + data={"messages": [{"role": "user", "content": "look up persimmon"}]}, + user_api_key_dict=UserAPIKeyAuth(), + response=self._server_tool_call_response(), + ) + + assert [call.kwargs["json"]["messages"] for call in post_mock.call_args_list] == [ + [ + {"role": "user", "content": "look up persimmon"}, + {"role": "assistant", "content": '{"query": "persimmon"} {"city": "Paris"}'}, + ] + ] + + @pytest.mark.asyncio + async def test_post_call_redact_clears_server_tool_call_arguments(self): + g = _make_guardrail(event_hook="post_call", on_flagged_action="monitor") + cisco_resp = _redact_response(sanitized_text="[REDACTED]") + + with _patch_inspection_post(g, AsyncMock(return_value=cisco_resp)): + result = await g.async_post_call_success_hook( + data={"messages": [{"role": "user", "content": "look up persimmon"}]}, + user_api_key_dict=UserAPIKeyAuth(), + response=self._server_tool_call_response(), + ) + + message = result.choices[0].message + assert message.provider_specific_fields == { + "server_tool_calls": [ + {"id": "srvtoolu_01", "type": "function", "function": {"name": "web_search", "arguments": "{}"}} + ] + } + assert message.tool_calls[0].function.arguments == "{}" + class TestCiscoAIDefenseToolDefinitionBypass: diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_lasso.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_lasso.py index b8a219ba863..6b6849f4f49 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_lasso.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_lasso.py @@ -1,3 +1,4 @@ +import json import os import uuid from typing import Final @@ -1549,3 +1550,144 @@ async def test_api_error_handling_legacy_messages( assert "Failed to verify request safety with Lasso API" in str(excinfo.value) assert error_message in str(excinfo.value) assert route.called + + +def _reply_with_web_search_call( + arguments: str, tool_calls: list[dict[str, object]] | None = None +) -> litellm.ModelResponse: + return litellm.ModelResponse( + choices=[ + litellm.Choices( + index=0, + finish_reason="tool_calls", + message=litellm.Message( + role="assistant", + content=None, + tool_calls=tool_calls, + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": arguments}, + } + ] + }, + ), + ) + ] + ) + + +@pytest.mark.asyncio +async def test_post_call_scan_sends_server_tool_calls_as_tool_use_entries( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + lasso_guardrail: Final = LassoGuardrail( + lasso_api_key="test-key", + user_id="test-user", + conversation_id="test-conversation", + guardrail_name="test-guard", + event_hook="post_call", + default_on=True, + ) + route: Final = respx_mock.post("https://server.lasso.security/gateway/v3/classify").mock( + return_value=httpx.Response(200, json={"violations_detected": False, "deputies": {}, "findings": {}}) + ) + response: Final = _reply_with_web_search_call( + '{"query": "persimmon"}', + tool_calls=[ + { + "id": "call_01", + "type": "function", + "function": {"name": "lookup_order", "arguments": '{"order_id": "A-17"}'}, + } + ], + ) + + await lasso_guardrail.async_post_call_success_hook( + data={"messages": [{"role": "user", "content": "search persimmons"}]}, + user_api_key_dict=UserAPIKeyAuth(), + response=response, + ) + + assert json.loads(route.calls.last.request.content) == { + "messages": [ + { + "role": "model", + "content": { + "type": "tool_use", + "id": "srvtoolu_01", + "name": "web_search", + "input": {"query": "persimmon"}, + }, + }, + { + "role": "model", + "content": { + "type": "tool_use", + "id": "call_01", + "name": "lookup_order", + "input": {"order_id": "A-17"}, + }, + }, + ], + "messageType": "COMPLETION", + "source": {"type": "litellm"}, + "userId": "test-user", + "sessionId": "test-conversation", + } + + +@pytest.mark.asyncio +async def test_post_call_masking_rewrites_server_tool_call_arguments( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + lasso_guardrail: Final = LassoGuardrail( + lasso_api_key="test-key", + user_id="test-user", + conversation_id="test-conversation", + mask=True, + guardrail_name="test-guard", + event_hook="post_call", + default_on=True, + ) + respx_mock.post("https://server.lasso.security/gateway/v3/classifix").mock( + return_value=httpx.Response( + 200, + json={ + "violations_detected": True, + "deputies": {"pattern-detection": True}, + "findings": {"pattern-detection": [{"action": "AUTO_MASKING", "severity": "HIGH"}]}, + "messages": [ + { + "role": "model", + "content": { + "type": "tool_use", + "id": "srvtoolu_01", + "name": "web_search", + "input": {"query": " persimmon"}, + }, + } + ], + }, + ) + ) + + result: Final = await lasso_guardrail.async_post_call_success_hook( + data={"messages": [{"role": "user", "content": "search persimmons"}]}, + user_api_key_dict=UserAPIKeyAuth(), + response=_reply_with_web_search_call('{"query": "jane@example.com persimmon"}'), + ) + + assert result.choices[0].message.provider_specific_fields == { + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": " persimmon"}'}, + } + ] + } diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py index 3da77a8707e..7e11505d0d0 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py @@ -893,6 +893,55 @@ class TestRestoration: assert result.choices[0].message.content == "a@b.com" + @pytest.mark.asyncio + async def test_server_tool_call_arguments_are_restored(self): + guardrail, _ = _shielded({"[EMAIL_1]": "a@b.com", "[URL_1]": "https://b.com/a"}) + response = ModelResponse( + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "[EMAIL_1] persimmon"}'}, + }, + { + "id": "srvtoolu_02", + "type": "function", + "function": {"name": "web_fetch", "arguments": '{"url": "[URL_1]"}'}, + }, + ] + }, + ), + ) + ] + ) + + result = await guardrail.async_post_call_success_hook( + data={"messages": []}, user_api_key_dict=None, response=response + ) + + assert result.choices[0].message.provider_specific_fields == { + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "a@b.com persimmon"}'}, + }, + { + "id": "srvtoolu_02", + "type": "function", + "function": {"name": "web_fetch", "arguments": '{"url": "https://b.com/a"}'}, + }, + ] + } + @pytest.mark.asyncio async def test_responses_api_shape_is_restored(self): """The Responses API reply carries output items, not choices. diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py index 25d2556bf26..c7d7cb5da54 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py @@ -2072,6 +2072,63 @@ class TestCompletionResponseTextPartsToolCalls: sent_text = mock_check.call_args.kwargs["text"] assert '{"credit_card": "4111-1111-1111-1111"}' in sent_text + @pytest.mark.asyncio + async def test_post_call_hook_sends_server_tool_call_args_to_purview(self): + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.types.utils import Choices, Message, ModelResponse + + def graph(request: httpx.Request) -> httpx.Response: + if request.url.host == "login.microsoftonline.com": + return httpx.Response(200, json={"access_token": "graph-token", "expires_in": 3600}) + if not request.url.path.endswith("/processContent"): + return httpx.Response(200, json={"value": []}, headers={"ETag": "scope-etag"}) + return httpx.Response(200, json={"protectionScopeState": "unchanged", "policyActions": []}) + + transport: Final = Mock(side_effect=graph) + guardrail: Final = _make_guardrail() + guardrail.async_handler = AsyncHTTPHandler(transport=httpx.MockTransport(transport)) + response: Final = ModelResponse( + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + } + ], + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + ] + }, + ), + ) + ], + ) + + await guardrail.async_post_call_success_hook( + data={}, + user_api_key_dict=UserAPIKeyAuth(api_key="test", user_id="user-123"), + response=response, + ) + + scanned: Final = [ + json.loads(call.args[0].content)["contentToProcess"]["contentEntries"][0]["content"]["data"] + for call in transport.call_args_list + if call.args[0].url.path.endswith("/processContent") + ] + assert scanned == ['{"query": "persimmon"}\n\n---\n\n{"city": "Paris"}'] + def test_responses_api_function_call_args_included(self): """Function-call arguments in ``ResponsesAPIResponse.output`` must be DLP-scanned.""" from litellm.types.llms.openai import ResponsesAPIResponse diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_pangea.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_pangea.py index 4e31a3c2055..0c5b8747e46 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_pangea.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_pangea.py @@ -1,5 +1,7 @@ from unittest.mock import AsyncMock, patch +from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch + import httpx import pytest from fastapi import HTTPException @@ -11,6 +13,7 @@ from litellm.proxy.guardrails.guardrail_hooks.pangea.pangea import ( ) from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 from litellm.types.guardrails import GuardrailEventHooks, LitellmParams +from litellm.types.responses.main import build_web_search_call from litellm.types.utils import Choices, Message, ModelResponse @@ -368,3 +371,66 @@ async def test_pangea_ai_guard_response_transformed(pangea_guardrail): ) assert response.choices[0]["message"]["content"] == "Yes, here is an SSN: " + + +def _claude_fetch_call(url): + return { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_fetch", "arguments": '{"url": "%s"}' % url}, + } + + +def _with_masked_tool_calls(choice): + message = choice["message"] + masked_calls = [_claude_fetch_call("") for _ in message["tool_calls"]] + return {**choice, "message": {**message, "tool_calls": masked_calls}} + + +@pytest.mark.asyncio +async def test_pangea_ai_guard_response_masks_claude_server_tool_call_inputs(pangea_guardrail): + data = {"messages": [{"role": "user", "content": "Read https://example.com/"}]} + fetch = build_web_search_call("srvtoolu_01", {"url": "https://example.com/"}, None, tool_name="web_fetch") + sent_choices = [] + + async def _mask_tool_call_urls(_handler, **kwargs): + choices = kwargs["json"]["input"]["choices"] + sent_choices.extend(choices) + return httpx.Response( + status_code=200, + json={ + "result": { + "blocked": False, + "transformed": True, + "output": {"choices": [_with_masked_tool_calls(choice) for choice in choices]}, + } + }, + request=httpx.Request(method="POST", url=kwargs["url"]), + ) + + with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new=_mask_tool_call_urls): + response = await pangea_guardrail.async_post_call_success_hook( + data=data, + user_api_key_dict=None, + response=ModelResponse( + choices=[ + Choices( + message=Message( + role="assistant", + content="Done", + provider_specific_fields={ + "server_tool_calls": [_claude_fetch_call("https://example.com/")], + "web_search_calls": [fetch], + }, + ) + ) + ] + ), + ) + + assert sent_choices[0]["message"]["tool_calls"] == [_claude_fetch_call("https://example.com/")] + message = response.choices[0]["message"] + assert message["tool_calls"] is None + assert message["provider_specific_fields"]["server_tool_calls"] == [_claude_fetch_call("")] + hosted_copy = ResponseFunctionWebSearch.model_validate(message["provider_specific_fields"]["web_search_calls"][0]) + assert hosted_copy.action.url == "" diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py index a41331230c2..834bd4ce224 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py @@ -6513,5 +6513,86 @@ class TestPanwAirsToolCallBlockMaskedDataRouting: assert error["scan_id"] == "scan-tool-2" +class TestPanwAirsServerToolCalls: + SERVER_TOOL_CALL: Final = { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "persimmon"}'}, + } + + @classmethod + def _response(cls, *client_tool_calls: ChatCompletionMessageToolCall) -> ModelResponse: + return ModelResponse( + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=list(client_tool_calls) or None, + provider_specific_fields={"server_tool_calls": [copy.deepcopy(cls.SERVER_TOOL_CALL)]}, + ), + ) + ], + model="claude-opus-4-1", + ) + + @staticmethod + def _airs(scan_result: Mapping[str, object]) -> tuple[PanwPrismaAirsHandler, MagicMock]: + transport: Final = MagicMock(side_effect=lambda request: httpx.Response(200, json=scan_result)) + handler: Final = make_handler( + mask_response_content=True, + http_client=AsyncHTTPHandler(transport=httpx.MockTransport(transport)), + ) + return handler, transport + + @pytest.mark.asyncio + async def test_post_call_scan_sends_server_tool_call_query_to_airs(self, user_api_key_dict): + handler, transport = self._airs({"action": "allow", "category": "benign", "scan_id": "s1"}) + response: Final = self._response( + ChatCompletionMessageToolCall( + id="call_1", + type="function", + function=Function(name="get_weather", arguments='{"city": "Paris"}'), + ) + ) + + await handler.async_post_call_success_hook( + data={"model": "claude-opus-4-1", "litellm_call_id": "call-srv", "metadata": {}}, + user_api_key_dict=user_api_key_dict, + response=response, + ) + + sent_contents: Final = [json.loads(call.args[0].content)["contents"] for call in transport.call_args_list] + assert sent_contents == [[{"response": '{"query": "persimmon"} {"city": "Paris"}'}]] + + @pytest.mark.asyncio + async def test_post_call_masking_rewrites_server_tool_call_arguments(self, user_api_key_dict): + masked: Final = '{"query": "*********"}' + handler, _ = self._airs( + { + "action": "block", + "category": "sensitive_data", + "scan_id": "s1", + "response_masked_data": {"data": masked}, + } + ) + + result: Final = await handler.async_post_call_success_hook( + data={"model": "claude-opus-4-1", "litellm_call_id": "call-srv-mask", "metadata": {}}, + user_api_key_dict=user_api_key_dict, + response=self._response(), + ) + + message: Final = result.choices[0].message + assert message.provider_specific_fields == { + "server_tool_calls": [ + {"id": "srvtoolu_01", "type": "function", "function": {"name": "web_search", "arguments": masked}} + ] + } + assert message.tool_calls is None + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py index e312378969a..43ed53cca1d 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -1781,6 +1781,48 @@ async def test_pii_tokens_in_metadata_used_for_unmasking(): assert response.choices[0].message.content == "Hello John, how can I help you?" +@pytest.mark.asyncio +async def test_output_unmasking_restores_server_tool_call_arguments(mock_user_api_key): + guardrail: Final = OPTIONAL_PresidioPIIMasking(mock_testing=True, output_parse_pii=True) + response: Final = ModelResponse( + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": " persimmon"}'}, + } + ] + }, + ), + ) + ] + ) + + result: Final = await guardrail.async_post_call_success_hook( + data={"metadata": {"pii_tokens": {"": "John Smith"}}}, + user_api_key_dict=mock_user_api_key, + response=response, + ) + + assert result.choices[0].message.provider_specific_fields == { + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "John Smith persimmon"}'}, + } + ] + } + + @pytest.mark.parametrize( "initial_hook", ["pre_call", "during_call", "pre_mcp_call"], diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_repelloai.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_repelloai.py index 77883e9af0e..246636e016f 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_repelloai.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_repelloai.py @@ -852,6 +852,59 @@ class TestRepelloAIPostCall: in captured["json"]["scan_data"]["response"] ) + @pytest.mark.asyncio + async def test_server_tool_call_arguments_scanned(self, monkeypatch): + guardrail = _guardrail(event_hook="post_call") + response = ModelResponse( + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=[ + { + "id": "call_01", + "type": "function", + "function": { + "name": "lookup_order", + "arguments": '{"order_id": "A-17"}', + }, + } + ], + provider_specific_fields={ + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": { + "name": "web_search", + "arguments": '{"query": "persimmon"}', + }, + } + ] + }, + ), + ) + ] + ) + captured = {} + + async def capture(url, headers, json, **_kwargs): + captured["json"] = json + return _verdict_response("passed", url) + + monkeypatch.setattr(guardrail.async_handler, "post", capture) + await guardrail.async_post_call_success_hook( + data={"messages": [{"role": "user", "content": "q"}]}, + user_api_key_dict=UserAPIKeyAuth(), + response=response, + ) + assert captured["json"]["scan_data"] == { + "response": '{"query": "persimmon"}\n{"order_id": "A-17"}' + } + # ---------------------------------------------------------------------- # verdict handling: unknown / malformed responses must not fail open diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py index 21376068686..5bf61585be2 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py @@ -39,6 +39,14 @@ from litellm.types.utils import ( ) +_SERVER_WEB_SEARCH_CALL: Final = { + "id": "srvtoolu_01Search", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "weather in Paris"}'}, +} +_ALLOWED_BASH_CALL: Final = {"id": "toolu_01Bash", "type": "function", "function": {"name": "Bash", "arguments": "{}"}} + + class TestToolPermissionGuardrail: """Test class for Tool Permission Guardrail functionality""" @@ -1034,6 +1042,70 @@ class TestToolPermissionGuardrail: assert "Permission denied" in choice.message.content + def _server_search_response(self) -> ModelResponse: + return ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message={ + "content": "Searching.", + "tool_calls": [_ALLOWED_BASH_CALL], + "provider_specific_fields": {"server_tool_calls": [_SERVER_WEB_SEARCH_CALL]}, + }, + ) + ] + ) + + @pytest.mark.asyncio + async def test_denied_server_tool_call_is_blocked(self): + with patch.object(self.guardrail, "should_run_guardrail", return_value=True): + with pytest.raises(GuardrailRaisedException): + await self.guardrail.async_post_call_success_hook( + data={}, user_api_key_dict=UserAPIKeyAuth(), response=self._server_search_response() + ) + + @pytest.mark.asyncio + async def test_denied_server_tool_call_in_a_chat_stream_is_blocked(self): + async def _stream(): + yield ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta( + content="Searching.", + provider_specific_fields={"server_tool_calls": [{**_SERVER_WEB_SEARCH_CALL, "index": 0}]}, + ), + finish_reason="stop", + ) + ] + ) + + with patch.object(self.guardrail, "should_run_guardrail", return_value=True): + with pytest.raises(GuardrailRaisedException): + async for _ in self.guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_stream(), request_data={} + ): + pass + + @pytest.mark.asyncio + async def test_rewrite_mode_removes_the_denied_server_tool_call(self): + rewriting: Final = ToolPermissionGuardrail( + guardrail_name="test-tool-permission", + rules=self.test_rules, + default_action="deny", + on_disallowed_action="rewrite", + ) + + with patch.object(rewriting, "should_run_guardrail", return_value=True): + result: Final = await rewriting.async_post_call_success_hook( + data={}, user_api_key_dict=UserAPIKeyAuth(), response=self._server_search_response() + ) + + message: Final = result.choices[0].message + assert message.provider_specific_fields["server_tool_calls"] == [] + assert [tool_call.id for tool_call in message.tool_calls] == [_ALLOWED_BASH_CALL["id"]] + assert "Permission denied" in message.content + def test_modify_response_with_permission_errors_keeps_custom_tool_call_in_another_choice(self): denied_call = ChatCompletionMessageToolCall( id="call_bash", type="function", function=Function(name="Bash", arguments="{}") @@ -1063,6 +1135,7 @@ class TestToolPermissionGuardrail: ] assert kept == [[], ["call_patch"]] + class TestToolPermissionGuardrailIntegration: """Integration tests for Tool Permission Guardrail""" diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 8a3e942e148..1fd2d60d360 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -50,6 +50,7 @@ from litellm.proxy.mcp_server.guardrail_translation.handler import ( ) from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.responses.main import build_web_search_call from litellm.types.utils import ( CallTypes, CallTypesLiteral, @@ -1288,6 +1289,36 @@ class TestStreamingTransform: # end of stream (it can block), matching block_only. assert guardrail.received_tool_calls + @pytest.mark.asyncio + async def test_claude_server_tool_call_chunks_keep_their_provider_fields(self): + server_fields = { + "server_tool_calls": [ + { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_fetch", "arguments": '{"url": "https://example.com/"}'}, + } + ], + "web_search_calls": { + "srvtoolu_01": build_web_search_call( + "srvtoolu_01", {"url": "https://example.com/"}, None, tool_name="web_fetch" + ).model_dump() + }, + } + server_chunk = ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(content=None, provider_specific_fields=server_fields))], + ) + + out = await _drive_stream( + UnifiedLLMGuardrails(), + _StreamingTextGuardrail(), + [server_chunk, _stream_chunk("done", finish_reason="stop")], + ) + + delivered = [getattr(item.choices[0].delta, "provider_specific_fields", None) for item in out if item.choices] + assert server_fields in delivered + assert "".join(_delta_text(item) for item in out) == "DONE" + @pytest.mark.asyncio async def test_per_choice_finish_reason_when_choices_finish_in_different_chunks(self): """n>1: a choice finishing before the stream's last chunk keeps its own diff --git a/tests/unit/proxy/guardrails/test_pillar_guardrails.py b/tests/unit/proxy/guardrails/test_pillar_guardrails.py index c6be433399a..766c7b1b00c 100644 --- a/tests/unit/proxy/guardrails/test_pillar_guardrails.py +++ b/tests/unit/proxy/guardrails/test_pillar_guardrails.py @@ -34,6 +34,7 @@ from litellm.proxy.guardrails.guardrail_hooks.pillar.pillar import ( build_pillar_response_headers, ) from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 +from litellm.types.utils import Choices, Message, ModelResponse # ============================================================================ # FIXTURES @@ -669,6 +670,49 @@ async def test_post_call_hook_with_tool_calls( assert result == mock_llm_response_with_tools +@pytest.mark.asyncio +async def test_post_call_hook_blocks_a_flagged_claude_server_tool_call( + pillar_guardrail_instance, + sample_request_data, + user_api_key_dict, + pillar_clean_response, + pillar_flagged_response, +): + server_call = { + "id": "srvtoolu_01", + "type": "function", + "function": {"name": "web_fetch", "arguments": '{"url": "https://example.com/"}'}, + } + response = ModelResponse( + choices=[ + Choices( + message=Message( + role="assistant", + content="Fetched it", + provider_specific_fields={"server_tool_calls": [server_call]}, + ) + ) + ] + ) + + async def _pillar_flags_the_server_call(*_args, **kwargs): + scanned_tool_calls = kwargs["json"]["messages"][-1].get("tool_calls") or [] + return pillar_flagged_response if server_call in scanned_tool_calls else pillar_clean_response + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + side_effect=_pillar_flags_the_server_call, + ): + with pytest.raises(HTTPException) as excinfo: + await pillar_guardrail_instance.async_post_call_success_hook( + data=sample_request_data, + user_api_key_dict=user_api_key_dict, + response=response, + ) + + assert excinfo.value.status_code == 400 + + # ========================================================================= # HEADER CONFIGURATION TESTS # ========================================================================= diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 4250c78c669..aabf2e1d7cc 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -33,7 +33,7 @@ from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) from litellm.types.llms.base import HiddenParams -from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams +from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse from litellm.types.responses.main import OutputToolSearchCall, build_web_search_call from litellm.types.utils import ( ChatCompletionMessageToolCall, @@ -4993,6 +4993,159 @@ def test_function_call_tool_id_falls_back_to_unique_id_for_degenerate_call_id(): assert convert(openai)["id"] == "call_tokyo" +_WEATHER_TOOL_USE: Final = { + "type": "tool_use", + "id": "toolu_01Weather", + "name": "get_weather", + "input": {"city": "Paris"}, +} +_FETCHED_URL: Final = "https://example.com/" +_BLOCKED_URL: Final = "https://blocked.invalid/" +_ANTHROPIC_WEB_FETCH_TURN: Final[tuple[dict[str, object], ...]] = ( + {"type": "server_tool_use", "id": "srvtoolu_01Fetch", "name": "web_fetch", "input": {"url": _FETCHED_URL}}, + { + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_01Fetch", + "content": { + "type": "web_fetch_result", + "url": _FETCHED_URL, + "content": {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "Example"}}, + }, + }, + {"type": "server_tool_use", "id": "srvtoolu_02Fetch", "name": "web_fetch", "input": {"url": _BLOCKED_URL}}, + { + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_02Fetch", + "content": {"type": "web_fetch_tool_result_error", "error_code": "url_not_allowed"}, + }, + {"type": "text", "text": "The page is titled Example."}, + _WEATHER_TOOL_USE, +) + + +def _anthropic_turn_on_responses(content: Sequence[Mapping[str, object]]) -> ResponsesAPIResponse: + from unittest.mock import MagicMock + + import httpx + + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + anthropic_response: Final = { + "id": "msg_01Fetch", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": list(content), + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 100, "output_tokens": 50}, + } + raw_response: Final = MagicMock(spec=httpx.Response) + raw_response.json.return_value = anthropic_response + raw_response.status_code = 200 + raw_response.headers = {} + chat_response: Final = AnthropicConfig().transform_parsed_response( + completion_response=anthropic_response, + raw_response=raw_response, + model_response=ModelResponse(), + json_mode=False, + prefix_prompt=None, + ) + return LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="Use the web and check the weather in Paris.", + responses_api_request={ + "tools": [ + {"type": "web_search_20250305", "name": "web_search"}, + {"type": "web_fetch_20250910", "name": "web_fetch"}, + {"type": "function", "name": "get_weather", "parameters": {"type": "object"}}, + ] + }, + chat_completion_response=chat_response, + ) + + +def test_anthropic_web_fetch_is_a_hosted_open_page_call_on_responses() -> None: + response: Final = _anthropic_turn_on_responses(_ANTHROPIC_WEB_FETCH_TURN) + + assert [ + (item.id, item.status, item.action.model_dump()) for item in response.output if item.type == "web_search_call" + ] == [ + ("ws_srvtoolu_01Fetch", "completed", {"type": "open_page", "url": _FETCHED_URL}), + ("ws_srvtoolu_02Fetch", "failed", {"type": "open_page", "url": _BLOCKED_URL}), + ] + assert [item.name for item in response.output if item.type == "function_call"] == ["get_weather"] + + +@pytest.mark.parametrize( + ("server_tool_use", "expected_action"), + [ + ( + { + "type": "server_tool_use", + "id": "srvtoolu_01Pending", + "name": "web_fetch", + "input": {"url": _FETCHED_URL}, + }, + {"type": "open_page", "url": _FETCHED_URL}, + ), + ( + {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_search", "input": {"query": "python"}}, + {"type": "search", "query": "python"}, + ), + ], +) +def test_anthropic_server_web_call_without_a_result_is_an_in_progress_hosted_call_on_responses( + server_tool_use: Mapping[str, object], expected_action: Mapping[str, object] +) -> None: + response: Final = _anthropic_turn_on_responses((server_tool_use, _WEATHER_TOOL_USE)) + + web_calls: Final = [item for item in response.output if item.type == "web_search_call"] + assert [(item.id, item.status) for item in web_calls] == [("ws_srvtoolu_01Pending", "in_progress")] + assert expected_action.items() <= web_calls[0].action.model_dump().items() + assert [item.name for item in response.output if item.type == "function_call"] == ["get_weather"] + + +def test_anthropic_code_execution_is_never_a_client_function_call_on_responses() -> None: + response: Final = _anthropic_turn_on_responses( + ( + { + "type": "server_tool_use", + "id": "srvtoolu_01Bash", + "name": "bash_code_execution", + "input": {"command": "ls"}, + }, + { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01Bash", + "content": { + "type": "bash_code_execution_result", + "stdout": "notes.txt\n", + "stderr": "", + "return_code": 0, + }, + }, + { + "type": "server_tool_use", + "id": "srvtoolu_02Edit", + "name": "text_editor_code_execution", + "input": {"command": "view", "path": "notes.txt"}, + }, + { + "type": "server_tool_use", + "id": "srvtoolu_03Bash", + "name": "bash_code_execution", + "input": {"command": "pwd"}, + }, + _WEATHER_TOOL_USE, + ) + ) + + assert [(item.type, getattr(item, "call_id", item.id)) for item in response.output if item.type != "message"] == [ + ("code_interpreter_call", "srvtoolu_01Bash"), + ("function_call", "toolu_01Weather"), + ] + + class TestHostedWebSearchReplay: def test_emitted_hosted_search_output_round_trips_with_client_tool_result(self) -> None: search_result: Final = { @@ -5063,6 +5216,75 @@ class TestHostedWebSearchReplay: assert messages[2]["content"] == "Paris is sunny." assert history == original + @pytest.mark.parametrize( + "leading_text", ((), ({"type": "text", "text": "Let me look that up."},)), ids=("alone", "after_text") + ) + @pytest.mark.parametrize( + ("server_tool_use", "expected_input"), + ( + ( + {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_fetch", "input": {"url": _FETCHED_URL}}, + {"url": _FETCHED_URL}, + ), + ( + {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_search", "input": {"query": "python"}}, + {"query": "python"}, + ), + ), + ids=("web_fetch", "web_search"), + ) + def test_pending_claude_web_call_replays_to_anthropic_as_its_own_server_tool_use( + self, + leading_text: tuple[Mapping[str, object], ...], + server_tool_use: Mapping[str, object], + expected_input: Mapping[str, object], + ) -> None: + response: Final = _anthropic_turn_on_responses((*leading_text, server_tool_use, _WEATHER_TOOL_USE)) + history: Final = [ + {"role": "user", "content": "Use the web and check the weather in Paris."}, + *(item.model_dump(exclude_none=True) for item in response.output), + {"type": "function_call_output", "call_id": "toolu_01Weather", "output": "Paris is sunny."}, + ] + + anthropic_turns: Final = anthropic_messages_pt( + messages=LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=history, responses_api_request={} + ), + model="claude-sonnet-5-5", + llm_provider="anthropic", + ) + + assert [turn["role"] for turn in anthropic_turns] == ["user", "assistant", "user"] + assert [block for block in anthropic_turns[1]["content"] if block["type"] != "text"] == [ + {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": server_tool_use["name"], "input": expected_input}, + {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}}, + ] + assert [(block["type"], block["tool_use_id"]) for block in anthropic_turns[2]["content"]] == [ + ("tool_result", "toolu_01Weather") + ] + + @pytest.mark.parametrize( + ("call_id", "status"), + (("ws_srvtoolu_01Fetch", "completed"), ("ws_68a1b2c3d4", "in_progress"), ("srvtoolu_01Fetch", "in_progress")), + ids=("completed-claude-call", "pending-non-claude-call", "pending-unprefixed-id"), + ) + def test_web_call_that_is_not_a_pending_claude_call_replays_as_text_only( + self, call_id: str, status: Literal["completed", "in_progress"] + ) -> None: + search: Final = ResponseFunctionWebSearch( + id=call_id, type="web_search_call", status=status, action=ActionOpenPage(type="open_page", url=_FETCHED_URL) + ) + + messages: Final = ( + LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=search.model_dump(exclude_none=True) + ) + ) + + assert messages == [ + {"role": "assistant", "content": "Hosted web search: " + search.model_dump_json(exclude_none=True)} + ] + @pytest.mark.parametrize( "action", ( diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 86773dc7ddf..e40b3febd3f 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -356,6 +356,198 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to ) +_STREAMED_WEB_FETCH_CALL: Final[tuple[dict[str, object], ...]] = ( + { + "type": "message_start", + "message": { + "id": "msg_01Fetch", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "server_tool_use", "id": "srvtoolu_01Fetch", "name": "web_fetch"}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"url": "https://example.com/"}'}, + }, + {"type": "content_block_stop", "index": 0}, +) +_STREAMED_WEB_FETCH_RESULT: Final[tuple[dict[str, object], ...]] = ( + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_01Fetch", + "content": {"type": "web_fetch_result", "url": "https://example.com/"}, + }, + }, + {"type": "content_block_stop", "index": 1}, +) + + +def _streamed_text_then_weather_call(first_index: int) -> tuple[dict[str, object], ...]: + return ( + {"type": "content_block_start", "index": first_index, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": first_index, "delta": {"type": "text_delta", "text": "Fetched."}}, + {"type": "content_block_stop", "index": first_index}, + { + "type": "content_block_start", + "index": first_index + 1, + "content_block": {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {}}, + }, + { + "type": "content_block_delta", + "index": first_index + 1, + "delta": {"type": "input_json_delta", "partial_json": '{"city": "Paris"}'}, + }, + {"type": "content_block_stop", "index": first_index + 1}, + { + "type": "message_delta", + "delta": {"stop_reason": "tool_use", "stop_sequence": None}, + "usage": {"output_tokens": 20}, + }, + ) + + +def _anthropic_web_fetch_stream_events(fetch_finished: bool) -> tuple[dict[str, object], ...]: + if fetch_finished: + return (*_STREAMED_WEB_FETCH_CALL, *_STREAMED_WEB_FETCH_RESULT, *_streamed_text_then_weather_call(2)) + return (*_STREAMED_WEB_FETCH_CALL, *_streamed_text_then_weather_call(1)) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize( + ("fetch_finished", "expected_status", "expected_search_events"), + [ + pytest.param( + True, + "completed", + [ + ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, + ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, + ], + id="fetch_finished_in_this_turn", + ), + pytest.param( + False, + "in_progress", + [ + ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, + ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, + ], + id="fetch_still_waiting_on_its_result", + ), + ], +) +@pytest.mark.asyncio +async def test_streamed_anthropic_web_fetch_is_a_hosted_open_page_call( + sync_mode: bool, + fetch_finished: bool, + expected_status: str, + expected_search_events: list[ResponsesAPIStreamEvents], +) -> None: + from litellm.llms.anthropic.chat.handler import ModelResponseIterator + + parser: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=sync_mode, json_mode=False) + chunks: Final = [parser.chunk_parser(event) for event in _anthropic_web_fetch_stream_events(fetch_finished)] + iterator: Final = LiteLLMCompletionStreamingIterator( + model="claude-sonnet-5-5", + litellm_custom_stream_wrapper=_FakeStreamWrapper(chunks), + request_input="Fetch example.com and check the weather in Paris.", + responses_api_request={ + "tools": [ + {"type": "web_fetch_20250910", "name": "web_fetch"}, + {"type": "function", "name": "get_weather", "parameters": {"type": "object"}}, + ] + }, + custom_llm_provider="anthropic", + ) + + events: Final = await _collect_events(iterator, sync_mode) + + completed: Final = next( + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + assert [ + (item.id, item.status, item.action.model_dump()) + for item in completed.response.output + if item.type == "web_search_call" + ] == [("ws_srvtoolu_01Fetch", expected_status, {"type": "open_page", "url": "https://example.com/"})] + assert [ + event.type + for event in events + if getattr(event, "item_id", getattr(getattr(event, "item", None), "id", None)) == "ws_srvtoolu_01Fetch" + ] == expected_search_events + assert [item.name for item in completed.response.output if item.type == "function_call"] == ["get_weather"] + assert not any( + event.item.name == "web_fetch" + for event in events + if getattr(event, "type", None) == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED + and event.item.type == "function_call" + ) + + +_STREAMED_PENDING_CODE_EXECUTION: Final[tuple[dict[str, object], ...]] = ( + _STREAMED_WEB_FETCH_CALL[0], + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "server_tool_use", "id": "srvtoolu_01Bash", "name": "bash_code_execution"}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"command": "ls"}'}, + }, + {"type": "content_block_stop", "index": 0}, + *_streamed_text_then_weather_call(1), +) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_streamed_anthropic_code_execution_is_never_a_client_function_call(sync_mode: bool) -> None: + from litellm.llms.anthropic.chat.handler import ModelResponseIterator + + parser: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=sync_mode, json_mode=False) + iterator: Final = LiteLLMCompletionStreamingIterator( + model="claude-sonnet-5-5", + litellm_custom_stream_wrapper=_FakeStreamWrapper( + [parser.chunk_parser(event) for event in _STREAMED_PENDING_CODE_EXECUTION] + ), + request_input="List the files, then check the weather in Paris.", + responses_api_request={ + "tools": [ + {"type": "code_execution_20250825", "name": "code_execution"}, + {"type": "function", "name": "get_weather", "parameters": {"type": "object"}}, + ] + }, + custom_llm_provider="anthropic", + ) + + events: Final = await _collect_events(iterator, sync_mode) + + item_events: Final = [event for event in events if getattr(event, "item", None) is not None] + assert {event.item.call_id for event in item_events if event.item.type == "function_call"} == {"toolu_01Weather"} + completed: Final = next( + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + assert [item.call_id for item in completed.response.output if item.type == "function_call"] == ["toolu_01Weather"] + + def test_tool_calls_present_only_in_final_response_are_emitted_before_completed(): iterator = LiteLLMCompletionStreamingIterator( model="test-model", diff --git a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts index 78372fe56d9..a6693794df6 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts +++ b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts @@ -131,6 +131,52 @@ describe("ToolsSection utils", () => { expect(result[1].callData).toBeUndefined(); }); + it("should mark Claude server tools as called from provider_specific_fields.server_tool_calls", () => { + const log: Partial = { + request_id: "test-server-tools", + proxy_server_request: { + tools: [ + { type: "web_search_20250305", name: "web_search", max_uses: 3 }, + { type: "function", function: { name: "get_weather", description: "Get weather" } }, + { type: "function", function: { name: "send_email", description: "Send email" } }, + ], + }, + response: { + choices: [ + { + message: { + tool_calls: [ + { + id: "toolu_01Weather", + type: "function", + function: { name: "get_weather", arguments: '{"location": "Paris"}' }, + }, + ], + provider_specific_fields: { + server_tool_calls: [ + { + id: "srvtoolu_01Search", + type: "function", + function: { name: "web_search", arguments: '{"query": "paris weather"}' }, + }, + ], + }, + }, + }, + ], + }, + }; + + const result = parseToolsFromLog(log as LogEntry); + + expect(result.map((tool) => [tool.name, tool.called, tool.callData?.id])).toEqual([ + ["web_search", true, "srvtoolu_01Search"], + ["get_weather", true, "toolu_01Weather"], + ["send_email", false, undefined], + ]); + expect(result[0].callData?.arguments).toEqual({ query: "paris weather" }); + }); + it("should handle string format request and response", () => { const log: Partial = { request_id: "test-5", diff --git a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts index 5e76a09c155..b1b2e6feb99 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts +++ b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts @@ -42,6 +42,24 @@ function extractToolsFromRequest(log: LogEntry): ToolDefinition[] { return []; } +interface ChatMessageToolCallFields { + tool_calls?: unknown; + provider_specific_fields?: { server_tool_calls?: unknown } | null; +} + +function toolCallList(value: unknown): ToolCall[] { + return Array.isArray(value) ? value : []; +} + +function extractChatMessageToolCalls(message: ChatMessageToolCallFields | undefined): ToolCall[] | null { + const serverToolCalls = message?.provider_specific_fields?.server_tool_calls; + const clientToolCalls = message?.tool_calls; + if (!Array.isArray(serverToolCalls) && !Array.isArray(clientToolCalls)) { + return null; + } + return [...toolCallList(serverToolCalls), ...toolCallList(clientToolCalls)]; +} + /** * Extract tool calls from response data */ @@ -53,10 +71,9 @@ function extractToolCallsFromResponse(log: LogEntry): ToolCall[] { // OpenAI format: response.choices[0].message.tool_calls const choices = responseData.choices; if (Array.isArray(choices) && choices.length > 0) { - const firstChoice = choices[0]; - const message = firstChoice.message; - if (message && Array.isArray(message.tool_calls)) { - return message.tool_calls; + const chatToolCalls = extractChatMessageToolCalls(choices[0].message); + if (chatToolCalls) { + return chatToolCalls; } }