scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-01 07:03:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 129

Merge b062c49e41 into a392a39627

Browse source
This commit is contained in:
JeongCheolLee 2026-10-31 06:29:37 +00:00 • committed by GitHub
commit 448c10cd8d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
73 changed files with 3970 additions and 328 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
litellm/integrations/arize/_utils.py
View file

@ -14,6 +14,7 @@ from litellm.litellm_core_utils.redact_messages import (
should_redact_message_logging,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls
from litellm.types.utils import CallTypes, StandardLoggingMCPToolCall, StandardLoggingPayload
if TYPE_CHECKING:
@ -663,12 +664,11 @@ def _to_plain_dict(value):
def _get_tool_calls(message) -> list | None:
"""Return ``message.tool_calls`` only when it's a non-empty list.
Works for dicts and Pydantic message objects via ``_safe_get``.
"""Return the message's tool calls, Claude's server tool calls first, or
``None`` when there are none. Works for dicts and Pydantic message objects.
"""
tool_calls: Final = _safe_get(message, "tool_calls")
return tool_calls if isinstance(tool_calls, list) and tool_calls else None
tool_calls: Final = tool_calls_with_server_tool_calls(message)
return list(tool_calls) if tool_calls else None
class _NormalizedToolCallFunction(TypedDict):

6
litellm/integrations/datadog/datadog_llm_obs.py
View file

@ -44,6 +44,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
@ -315,7 +316,10 @@ def _to_dd_tool_calls(message: Mapping[str, object]) -> tuple[ToolCall, ...]:
tool_id=_text_field(tool_call, "id"),
type=_text_field(tool_call, "type", "function"),
)
for tool_call in (raw_tool_calls if isinstance(raw_tool_calls, list) else ())
for tool_call in (
*anthropic_server_tool_call_params(message),
*(raw_tool_calls if isinstance(raw_tool_calls, list) else ()),
)
if isinstance(tool_call, dict)
for function in [_mapping_field(tool_call, "function")]
)

36
litellm/integrations/helicone.py
View file

@ -10,6 +10,23 @@ from litellm.integrations.helicone_mock_client import (
create_mock_helicone_client,
should_use_helicone_mock,
)
from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls
from litellm.types.utils import ChatCompletionMessageCustomToolCall, ChatCompletionMessageToolCall
def _tool_use_block(
tool_call: ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall,
) -> dict[str, object]:
match tool_call:
case ChatCompletionMessageCustomToolCall(custom=custom):
return {"type": "tool_use", "id": tool_call.id, "name": custom.name, "input": custom.input}
case _:
return {
"type": "tool_use",
"id": tool_call.id,
"name": tool_call.function.name,
"input": tool_call.function.arguments,
}
class HeliconeLogger:
@ -57,23 +74,8 @@ class HeliconeLogger:
message = choice["message"]
content = []
if "tool_calls" in message and message["tool_calls"]:
for tool_call in message["tool_calls"]:
function = tool_call.get("function")
custom = tool_call.get("custom")
if not function and not custom:
continue
name, tool_input = (
(function["name"], function["arguments"]) if function else (custom["name"], custom["input"])
)
content.append(
{
"type": "tool_use",
"id": tool_call["id"],
"name": name,
"input": tool_input,
}
)
if tool_calls := tool_calls_with_server_tool_calls(message):
content = [_tool_use_block(tool_call) for tool_call in tool_calls]
elif "content" in message and message["content"]:
content = [{"type": "text", "text": message["content"]}]

5
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -26,6 +26,7 @@ from litellm.integrations.opentelemetry import (
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls
from litellm.secret_managers.main import str_to_bool
from litellm.types.integrations.langfuse_otel import (
LangfuseSpanAttributes,
@ -605,8 +606,8 @@ def _extract_choices_output(response_obj: _Gettable) -> str | None:
message: Final = _as_gettable(first_choice.get("message", {}))
if message is None:
return None
tool_calls: Final = message.get("tool_calls")
if isinstance(tool_calls, list) and tool_calls:
tool_calls: Final = tool_calls_with_server_tool_calls(message)
if tool_calls:
transformed_tool_calls: Final = [
entry for tool_call in tool_calls if (entry := _transformed_tool_call(response_obj, tool_call)) is not None
]

5
litellm/integrations/lunary.py
View file

@ -6,6 +6,7 @@ from datetime import datetime, timezone
from typing import TYPE_CHECKING, Final
import packaging
from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls
if TYPE_CHECKING:
from openai.types.chat.chat_completion_message_custom_tool_call import Custom
@ -67,8 +68,8 @@ def parse_messages(input):
}
# Only add tool_calls and function_call to res if they are set
if message.get("tool_calls"):
serialized["tool_calls"] = parse_tool_calls(message.get("tool_calls"))
if tool_calls := tool_calls_with_server_tool_calls(message):
serialized["tool_calls"] = parse_tool_calls(tool_calls)
return serialized

7
litellm/integrations/newrelic/newrelic.py
View file

@ -56,6 +56,7 @@ from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
from litellm.types.integrations.newrelic import NewRelicInitParams
from litellm.types.utils import Message, ModelResponse, StandardLoggingPayload
@ -479,11 +480,11 @@ class NewRelicLogger(CustomLogger):
content: Final = message.get("content")
# Handle tool calls
if message.get("tool_calls"):
if tool_calls := tool_calls_with_server_tool_calls(message):
try:
return json.dumps(message["tool_calls"])
return json.dumps([tool_call.model_dump() for tool_call in tool_calls])
except Exception:
return str(message["tool_calls"])
return str(tool_calls)
# Handle None or missing content
if content is None:

6
litellm/integrations/opentelemetry.py
View file

@ -42,6 +42,7 @@ from litellm.litellm_core_utils.service_tier_utils import (
get_requested_service_tier,
get_served_service_tier,
)
from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls
from litellm.secret_managers.main import get_secret_bool, str_to_bool
from litellm.types.integrations.otel_span_attributes import SpanAttributes
from litellm.types.services import ServiceLoggerPayload
@ -2736,9 +2737,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
for idx, choice in enumerate(response_obj.get("choices")):
if choice.get("finish_reason"):
message = choice.get("message")
tool_calls = message.get("tool_calls")
if tool_calls:
kv_pairs = OpenTelemetry._tool_calls_kv_pair(tool_calls)
if tool_calls := tool_calls_with_server_tool_calls(message):
kv_pairs = OpenTelemetry._tool_calls_kv_pair(list(tool_calls))
for key, value in kv_pairs.items():
self.safe_set_attribute(
span=span,

11
litellm/integrations/otel/mappers/utils.py
View file

@ -12,6 +12,8 @@ from typing import Final
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue
from litellm.integrations.otel.model.payloads import LLMCallSpanData, ToolDefinition
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
@dataclass(frozen=True, slots=True)
@ -141,12 +143,13 @@ def output_messages(data: LLMCallSpanData) -> list:
def message_tool_calls(message: object) -> tuple[MessageToolCall, ...]:
tool_call_values: Final = (
*anthropic_server_tool_call_params(message),
*parts_of(message_field(message, "tool_calls")),
)
if not isinstance(message, dict):
return ()
tool_calls: Final = message.get("tool_calls")
if not isinstance(tool_calls, (list, tuple)) or not tool_calls:
return ()
return tuple(tool_call for value in tool_calls if (tool_call := _message_tool_call(value)) is not None)
return tuple(tool_call for value in tool_call_values if (tool_call := _message_tool_call(value)) is not None)
def _message_tool_call(value: object) -> MessageToolCall | None:

29
litellm/integrations/otel/model/payloads.py
View file

@ -10,6 +10,8 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, ClassVar, Final, Literal, Protocol, cast
from urllib.parse import urlsplit
from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch
from pydantic import TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm.integrations.otel.model.metadata import (
@ -36,6 +38,7 @@ from litellm.integrations.otel.model.utils import (
as_str_tuple,
)
from litellm.integrations.otel.routing import RoutingAttributeValue, routing_decision_attributes
from litellm.types.llms.anthropic import claude_hosted_web_tool_call
# ``RequestIdentity`` and the request-metadata translation now live in
# :mod:`metadata`; re-exported here so existing ``model.payloads`` imports keep
@ -809,6 +812,7 @@ class _Choice(TypedDict):
_RESPONSES_TOOL_CALL_TYPES: Final = frozenset({"function_call", "custom_tool_call"})
_WEB_SEARCH_CALL: Final = TypeAdapter(ResponseFunctionWebSearch)
def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
@ -816,9 +820,13 @@ def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
items: Final = _dicts(response.get("output"))
messages: Final = tuple(item for item in items if item.get("type") == "message")
parts: Final = tuple(part for item in messages for part in _dicts(item.get("content")))
tool_calls: Final = tuple(
client_tool_calls: Final = tuple(
_responses_tool_call(item) for item in items if item.get("type") in _RESPONSES_TOOL_CALL_TYPES
)
tool_calls: Final = (
*(call for item in items if (call := _claude_web_search_tool_call(item)) is not None),
*client_tool_calls,
)
if not messages and not tool_calls:
return ()
message: Final[_AssistantMessage] = {
@ -827,7 +835,10 @@ def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
"refusal": _responses_parts_text(parts, "refusal", "refusal"),
"tool_calls": tool_calls or None,
}
choice: Final[_Choice] = {"message": message, "finish_reason": _responses_finish_reason(response, bool(tool_calls))}
choice: Final[_Choice] = {
"message": message,
"finish_reason": _responses_finish_reason(response, bool(client_tool_calls)),
}
return (choice,)
@ -981,6 +992,20 @@ def _responses_tool_call(item: Mapping[str, object]) -> _ToolCall:
return tool_call
def _claude_web_search_tool_call(item: Mapping[str, object]) -> _ToolCall | None:
if item.get("type") != "web_search_call":
return None
try:
search: Final = _WEB_SEARCH_CALL.validate_python(item)
except ValidationError:
return None
claude_call: Final = claude_hosted_web_tool_call(search)
if claude_call is None:
return None
tool_id, name, arguments = claude_call
return {"id": tool_id, "type": "function", "function": {"name": name, "arguments": arguments}}
def _responses_finish_reason(response: Mapping[str, object], has_tool_calls: bool) -> str | None:
status: Final = as_str(response.get("status"))
if status == "completed":

8
litellm/integrations/shadow_eval_logger.py
View file

@ -39,6 +39,7 @@ from litellm.litellm_core_utils.llm_judge import (
parse_json_verdict,
)
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls
from litellm.llms.base_llm.base_utils import type_to_response_format_param
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalDirection
@ -202,7 +203,7 @@ def _chat_final_text(response_obj: object) -> str:
if read is None:
return ""
prose: Final = extract_text_from_content(read("content"))
if not (read("tool_calls") or read("function_call")):
if not _tool_calls_list(read):
return prose
serialized: Final = _serialize_tool_calls(read)
return f"{prose} {serialized}".strip() if prose else serialized
@ -218,8 +219,9 @@ _RESPONSES_TOOL_CALL_TYPES: Final = frozenset(("function_call", "custom_tool_cal
def _tool_calls_list(read: Callable[[str], object]) -> tuple[object, ...]:
calls: Final = read("tool_calls")
listed: Final = tuple(calls) if isinstance(calls, Sequence) and not isinstance(calls, str) else ()
listed: Final = tool_calls_with_server_tool_calls(
{"tool_calls": read("tool_calls"), "provider_specific_fields": read("provider_specific_fields")}
)
single: Final = read("function_call")
return listed if listed else ((single,) if single is not None else ())

7
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -606,6 +606,8 @@ def get_content_from_model_response(response: ModelResponse | dict) -> str:
else:
new_response = response
from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls
content = ""
for choice in new_response.choices:
@ -613,9 +615,8 @@ def get_content_from_model_response(response: ModelResponse | dict) -> str:
content += choice.message.content if choice.message.content else ""
if choice.message.function_call:
content += choice.message.function_call.model_dump_json()
if choice.message.tool_calls:
for tc in choice.message.tool_calls:
content += tc.model_dump_json()
for tc in tool_calls_with_server_tool_calls(choice.message):
content += tc.model_dump_json()
elif isinstance(choice, StreamingChoices):
content += getattr(choice, "delta", {}).get("content", "") or ""
return content

40
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -1995,12 +1995,39 @@ def _tool_call_ids_with_a_kept_server_result(message: object) -> frozenset[str]:
return frozenset()
return frozenset(
tool_call_id
for tool_call in parts_of(message_field(message, "tool_calls"))
for tool_call in (
*parts_of(message_field(message, "tool_calls")),
*parts_of(message_field(provider_specific_fields, "server_tool_calls")),
)
if isinstance(tool_call_id := message_field(tool_call, "id"), str)
and tool_call_is_rebuilt_as_server_tool_use(tool_call_id, provider_specific_fields)
)
def _assistant_tool_calls_with_server_tool_calls(
message: ChatCompletionAssistantMessage,
) -> Sequence[ChatCompletionAssistantToolCall] | None:
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
tool_calls: Final = message.get("tool_calls")
server_tool_calls: Final = anthropic_server_tool_call_params(message)
if not server_tool_calls:
return tool_calls
client_tool_call_ids: Final = frozenset(tool_call["id"] for tool_call in tool_calls or ())
return (
*(tool_call for tool_call in server_tool_calls if tool_call["id"] not in client_tool_call_ids),
*(tool_calls or ()),
)
def _server_tool_call_ids(message: object) -> frozenset[str]:
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
return frozenset(
tool_call["id"] for tool_call in anthropic_server_tool_call_params(message) if tool_call["id"] is not None
)
def _server_tool_answers_replaced_after(messages: Sequence[object], index: int) -> frozenset[str]:
kept_server_result_ids: Final = _tool_call_ids_with_a_kept_server_result(messages[index])
if not kept_server_result_ids:
@ -2060,10 +2087,11 @@ _AnthropicToolInvokeItem: TypeAlias = AnthropicMessagesToolUseParam | dict[str,
def convert_to_anthropic_tool_invoke(
tool_calls: list[ChatCompletionAssistantToolCall],
tool_calls: Sequence[ChatCompletionAssistantToolCall],
web_search_results: Sequence[object] | None = None,
tool_results: Sequence[object] | None = None,
answered_tool_ids: Container[str] = frozenset(),
server_tool_call_ids: Container[str] = frozenset(),
) -> list[_AnthropicToolInvokeItem]:
"""
OpenAI tool invokes:
@ -2141,6 +2169,10 @@ def convert_to_anthropic_tool_invoke(
anthropic_tool_invoke.append(server_tool_result)
elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids:
continue
elif tool_id in server_tool_call_ids and tool_id not in answered_tool_ids:
anthropic_tool_invoke.append(
{"type": "server_tool_use", "id": tool_id, "name": tool_name, "input": tool_input}
)
else:
sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id)
_anthropic_tool_use_param = AnthropicMessagesToolUseParam(
@ -3100,7 +3132,7 @@ def anthropic_messages_pt(
# If so, we need to interleave thinking blocks with tool call groups
# to preserve the original content block ordering.
# Fixes: https://github.com/BerriAI/litellm/issues/23047
assistant_tool_calls = assistant_content_block.get("tool_calls")
assistant_tool_calls = _assistant_tool_calls_with_server_tool_calls(assistant_content_block)
_has_server_tool_calls = False
if assistant_tool_calls is not None:
for _tc in assistant_tool_calls:
@ -3133,6 +3165,7 @@ def anthropic_messages_pt(
web_search_results=_web_search_results_tc,
tool_results=_tool_results_tc,
answered_tool_ids=client_answered_tool_ids,
server_tool_call_ids=_server_tool_call_ids(assistant_content_block),
)
# Group tool invoke results into (server_tool_use, result) pairs
@ -3298,6 +3331,7 @@ def anthropic_messages_pt(
web_search_results=_web_search_results,
tool_results=_tool_results,
answered_tool_ids=client_answered_tool_ids,
server_tool_call_ids=_server_tool_call_ids(assistant_content_block),
)
# Prevent "tool_use ids must be unique" errors by filtering duplicates

63
litellm/litellm_core_utils/redact_messages.py
View file

@ -14,7 +14,7 @@ from collections.abc import Mapping, Sequence
from dataclasses import replace
from typing import TYPE_CHECKING, Any, Final
from pydantic import JsonValue, TypeAdapter, ValidationError
from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
import litellm
from litellm.constants import REDACTED_BY_LITELLM
@ -254,9 +254,63 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[
)
if key in fields
},
**(
{"server_tool_calls": _redacted_tool_call_dicts(fields["server_tool_calls"])}
if "server_tool_calls" in fields
else {}
),
**(
{"web_search_calls": _redacted_web_search_calls(fields["web_search_calls"])}
if "web_search_calls" in fields
else {}
),
}
def _redacted_tool_call_dict(call: object) -> object:
if (mapping := _as_mapping(call)) is None or (function := _as_mapping(mapping.get("function"))) is None:
return call
return {**mapping, "function": {**function, "arguments": REDACTED_BY_LITELLM}}
def _redacted_tool_call_dicts(calls: object) -> object:
return [_redacted_tool_call_dict(call) for call in calls] if isinstance(calls, list) else calls
_WEB_ACTION_TEXT_FIELDS: Final = ("query", "url", "pattern")
_WEB_ACTION_LIST_FIELDS: Final = ("queries", "sources")
def _web_action_redactions(action: Mapping[str, object]) -> Mapping[str, object]:
return {
**{key: REDACTED_BY_LITELLM for key in _WEB_ACTION_TEXT_FIELDS if action.get(key) is not None},
**{key: None for key in _WEB_ACTION_LIST_FIELDS if action.get(key) is not None},
}
def _redacted_web_search_action(action: object) -> object:
if isinstance(action, BaseModel):
return action.model_copy(update=dict(_web_action_redactions(action.model_dump())))
mapping: Final = _as_mapping(action)
return action if mapping is None else {**mapping, **_web_action_redactions(mapping)}
def _redacted_web_search_call(item: object) -> object:
if isinstance(item, BaseModel):
action: Final = getattr(item, "action", None)
return item if action is None else item.model_copy(update={"action": _redacted_web_search_action(action)})
mapping: Final = _as_mapping(item)
if mapping is None or mapping.get("action") is None:
return item
return {**mapping, "action": _redacted_web_search_action(mapping["action"])}
def _redacted_web_search_calls(calls: object) -> object:
if (mapping := _as_mapping(calls)) is not None:
return {key: _redacted_web_search_call(call) for key, call in mapping.items()}
return [_redacted_web_search_call(call) for call in calls] if isinstance(calls, list) else calls
def _redact_field_when_present(item: object, field: str) -> None:
if getattr(item, field, None) is not None:
setattr(item, field, REDACTED_BY_LITELLM)
@ -283,6 +337,8 @@ def _redact_responses_api_output(output_items):
output_item.arguments = REDACTED_BY_LITELLM
if output_type == "custom_tool_call" and hasattr(output_item, "input"):
output_item.input = REDACTED_BY_LITELLM
if output_type == "web_search_call" and getattr(output_item, "action", None) is not None:
output_item.action = _redacted_web_search_action(output_item.action)
_RESPONSE_ITEMS: Final = TypeAdapter(tuple[object, ...])
@ -337,6 +393,11 @@ def _redacted_responses_api_output_item(item: object, redacted_str: str) -> obje
if item_mapping.get("type") == "custom_tool_call" and "input" in item_mapping
else {}
),
**(
{"action": _redacted_web_search_action(item_mapping["action"])}
if item_mapping.get("type") == "web_search_call" and item_mapping.get("action") is not None
else {}
),
}

44
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -16,7 +16,7 @@ import json
from collections.abc import Iterable, Mapping, MutableSequence, Sequence
from copy import deepcopy
from dataclasses import dataclass
from itertools import chain, repeat
from itertools import chain, filterfalse, repeat
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast, overload, runtime_checkable
@ -26,6 +26,7 @@ from typing_extensions import ReadOnly, TypedDict, assert_never
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.common_utils import anthropic_server_tool_calls
from litellm.llms.anthropic.messages.adapters.transformation import (
OPENAI_MAX_TOOL_NAME_LENGTH,
LiteLLMAnthropicMessagesAdapter,
@ -205,17 +206,15 @@ class _WebSearchString:
value: str
def _streamed_web_search_result_blocks(response: ModelResponse) -> tuple[Mapping[str, object], ...]:
choice: Final = response.choices[0] if response.choices else None
fields: Final = as_str_mapping(choice.message.provider_specific_fields) if isinstance(choice, Choices) else None
blocks: Final = (
(as_str_mapping(item) for item in iter_items(fields.get("web_search_results")))
if fields is not None and isinstance(fields.get("web_search_results"), list)
else ()
)
return tuple(
deepcopy(block) for block in blocks if block is not None and block.get("type") == "web_search_tool_result"
)
def _legacy_hook_block(block: object) -> object:
if not isinstance(block, Mapping):
return block
fields: Final = _as_str_mapping(block)
return {**fields, "type": "tool_use"} if fields.get("type") == "server_tool_use" else fields
def _is_web_search_result_block(block: object) -> bool:
return isinstance(block, Mapping) and _as_str_mapping(block).get("type") == "web_search_tool_result"
def _web_search_results(block: Mapping[str, object]) -> Iterable[object]:
@ -585,11 +584,14 @@ class AnthropicMessagesHandler(BaseTranslation):
if not isinstance(response, ModelResponse):
return response
translated: Final = self.adapter.translate_openai_response_to_anthropic(response)
result_blocks: Final = _streamed_web_search_result_blocks(response)
if not result_blocks:
return translated
content: Final = translated.get("content") or ()
return {**translated, "content": [*result_blocks, *content]}
blocks: Final = tuple(map(_legacy_hook_block, translated.get("content") or ()))
return {
**translated,
"content": [
*filter(_is_web_search_result_block, blocks),
*filterfalse(_is_web_search_result_block, blocks),
],
}
@staticmethod
def _build_streaming_usage_response(
@ -1543,9 +1545,9 @@ class AnthropicMessagesHandler(BaseTranslation):
if built_response is not None and hasattr(built_response, "choices") and built_response.choices:
model_response: Final = cast(ModelResponse, built_response)
first_choice: Final = cast(Choices, model_response.choices[0])
tool_calls_list: Final = cast(
list[ChatCompletionMessageToolCall] | None,
first_choice.message.tool_calls,
tool_calls_list: Final = (
*anthropic_server_tool_calls(first_choice.message),
*cast(list[ChatCompletionMessageToolCall], first_choice.message.tool_calls or []),
)
string_so_far = first_choice.message.content
pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ())
@ -1559,7 +1561,7 @@ class AnthropicMessagesHandler(BaseTranslation):
if texts_to_check:
guardrail_inputs["texts"] = texts_to_check
if tool_calls_list:
guardrail_inputs["tool_calls"] = tool_calls_list
guardrail_inputs["tool_calls"] = list(tool_calls_list)
try:
prepared_request_data = self._prepare_request_data(

65
litellm/llms/anthropic/chat/handler.py
View file

@ -54,6 +54,7 @@ from litellm.types.llms.openai import (
ChatCompletionToolCallFunctionChunk,
)
from litellm.types.responses.main import (
HOSTED_WEB_TOOL_NAMES,
OutputCodeInterpreterCall,
build_code_interpreter_log_outputs,
build_web_search_call,
@ -754,6 +755,8 @@ class ModelResponseIterator:
self._server_tool_inputs: dict[str, object] = {}
self.tool_results: list[dict[str, Any]] = []
self._current_server_tool_id: str | None = None
self._current_server_tool_name: str = ""
self._server_tool_calls: tuple[ChatCompletionToolCallChunk, ...] = ()
self._container_id: str | None = None
@property
@ -824,10 +827,9 @@ class ModelResponseIterator:
return text, tool_use, thinking_blocks, provider_specific_fields, reasoning_content
self.content_blocks.append(content_block)
if "partial_json" in content_block["delta"]:
# Only emit tool calls if we're in a tool_use or server_tool_use block
# web_search_tool_result blocks also have input_json_delta but should not be treated as tool calls
# See: https://github.com/BerriAI/litellm/issues/17254
if self.current_content_block_type in ("tool_use", "server_tool_use"):
if self.current_content_block_type == "tool_use":
tool_use = cast(
ChatCompletionToolCallChunk,
{
@ -926,7 +928,7 @@ class ModelResponseIterator:
def _web_search_call_snapshot(self) -> dict[str, object]:
return dict(self._web_search_calls)
def _complete_web_search_call(self, result: dict[str, object]) -> None:
def _complete_web_search_call(self, result: dict[str, object], tool_name: str) -> None:
tool_use_id: Final = result.get("tool_use_id")
if not isinstance(tool_use_id, str) or tool_use_id not in self._web_search_calls:
return
@ -934,6 +936,7 @@ class ModelResponseIterator:
tool_id=tool_use_id,
tool_input=self._server_tool_inputs.get(tool_use_id, {}),
result=result,
tool_name=tool_name,
)
def _build_code_interpreter_results(self) -> list:
@ -1013,7 +1016,6 @@ class ModelResponseIterator:
content_block_start["content_block"]["type"] == "tool_use"
or content_block_start["content_block"]["type"] == "server_tool_use"
):
self.tool_index += 1
# Reverse-map the (sanitized) tool name back to the
# caller's original. No-op when the map is empty.
_stream_tool_name = content_block_start["content_block"]["name"]
@ -1022,35 +1024,39 @@ class ModelResponseIterator:
# Use empty string for arguments in content_block_start - actual arguments
# come in subsequent content_block_delta chunks and get accumulated.
# Using str(input) here would prepend '{}' causing invalid JSON accumulation.
tool_use = ChatCompletionToolCallChunk(
id=content_block_start["content_block"]["id"],
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=_stream_tool_name,
arguments="",
),
index=self.tool_index,
)
if content_block_start["content_block"]["type"] == "tool_use":
self.tool_index += 1
tool_use = ChatCompletionToolCallChunk(
id=content_block_start["content_block"]["id"],
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=_stream_tool_name,
arguments="",
),
index=self.tool_index,
)
# Track server tool use inputs for code_interpreter_results.
# The initial input in content_block_start is typically {}
# for streaming; the full input arrives via input_json_delta
# and is assembled at content_block_stop.
if content_block_start["content_block"]["type"] == "server_tool_use":
self._current_server_tool_id = content_block_start["content_block"]["id"]
self._current_server_tool_name = _stream_tool_name
tool_input: Final = content_block_start["content_block"].get("input", {})
self._server_tool_inputs[self._current_server_tool_id] = tool_input
if _stream_tool_name == "web_search":
if _stream_tool_name in HOSTED_WEB_TOOL_NAMES:
self._web_search_calls[self._current_server_tool_id] = build_web_search_call(
self._current_server_tool_id,
tool_input,
{"content": []},
status="in_progress",
tool_name=_stream_tool_name,
)
provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
# Include caller information if present (for programmatic tool calling)
if "caller" in content_block_start["content_block"]:
caller_data: Final = content_block_start["content_block"]["caller"]
if caller_data:
if caller_data and tool_use is not None:
tool_use["caller"] = cast(dict[str, Any], caller_data)
elif content_block_start["content_block"]["type"] == "redacted_thinking":
(
@ -1081,7 +1087,7 @@ class ModelResponseIterator:
# The full content comes in content_block_start, not in deltas
# See: https://github.com/BerriAI/litellm/issues/17737
self.web_search_results.append(content_block_start["content_block"])
self._complete_web_search_call(content_block_start["content_block"])
self._complete_web_search_call(content_block_start["content_block"], "web_search")
provider_specific_fields["web_search_results"] = self.web_search_results
provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
elif content_type == "web_fetch_tool_result":
@ -1089,7 +1095,9 @@ class ModelResponseIterator:
# The full content comes in content_block_start, not in deltas
# Fixes: https://github.com/BerriAI/litellm/issues/18137
self.web_search_results.append(content_block_start["content_block"])
self._complete_web_search_call(content_block_start["content_block"], "web_fetch")
provider_specific_fields["web_search_results"] = self.web_search_results
provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
else:
self.tool_results.append(content_block_start["content_block"])
provider_specific_fields["tool_results"] = self.tool_results
@ -1104,7 +1112,7 @@ class ModelResponseIterator:
# check if tool call content block - only for tool_use and server_tool_use blocks
if self.current_content_block_type in ("tool_use", "server_tool_use"):
is_empty: Final = self.check_empty_tool_call_args()
if is_empty:
if is_empty and self.current_content_block_type == "tool_use":
tool_use = ChatCompletionToolCallChunk(
id=None,
type="function",
@ -1128,6 +1136,29 @@ class ModelResponseIterator:
self._server_tool_inputs[self._current_server_tool_id] = json.loads(args)
except (json.JSONDecodeError, TypeError):
pass
server_tool_input: Final = self._server_tool_inputs.get(self._current_server_tool_id, {})
if self._current_server_tool_id in self._web_search_calls:
self._web_search_calls[self._current_server_tool_id] = build_web_search_call(
self._current_server_tool_id,
server_tool_input,
{"content": []},
status="in_progress",
tool_name=self._current_server_tool_name,
)
provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
self._server_tool_calls = (
*self._server_tool_calls,
ChatCompletionToolCallChunk(
id=self._current_server_tool_id,
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=self._current_server_tool_name,
arguments=json.dumps(server_tool_input),
),
index=len(self._server_tool_calls),
),
)
provider_specific_fields["server_tool_calls"] = list(self._server_tool_calls)
self._current_server_tool_id = None
if self.is_response_format_tool:
text = self._flush_response_format_buffer()

89
litellm/llms/anthropic/chat/transformation.py
View file

@ -7,6 +7,7 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, cast
import httpx
from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
@ -90,11 +91,11 @@ from litellm.types.llms.openai import (
ChatCompletionToolCallChunk,
ChatCompletionToolCallFunctionChunk,
ChatCompletionToolParam,
OpenAIChatCompletionFinishReason,
OpenAIMcpServerTool,
OpenAIWebSearchOptions,
)
from litellm.types.responses.main import (
HOSTED_WEB_TOOL_NAMES,
OutputCodeInterpreterCall,
build_code_interpreter_log_outputs,
build_web_search_call,
@ -166,6 +167,7 @@ _ANTHROPIC_TOOL_NAME_MAX_LEN: Final = 128
_DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
_TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_TOOLS_ADAPTER: Final = TypeAdapter(list[object])
_CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(tuple[object, ...])
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
@ -252,6 +254,42 @@ def _validated_defer_loading(value: object) -> bool:
raise ValueError("defer_loading must be a boolean") from exc
def _hosted_web_tool_use(block: object) -> tuple[str, str, object] | None:
match block:
case {
"type": "server_tool_use",
"name": str() as name,
"id": str() as call_id,
"input": object() as tool_input,
} if name in HOSTED_WEB_TOOL_NAMES:
return call_id, name, tool_input
case _:
return None
def _server_tool_use_id(block: object) -> str | None:
match block:
case {"type": "server_tool_use", "id": str() as call_id}:
return call_id
case _:
return None
def _server_tool_use_ids(completion_response: Mapping[str, object]) -> frozenset[str]:
blocks: Final = _CONTENT_BLOCKS_ADAPTER.validate_python(completion_response.get("content") or ())
return frozenset(filter(None, map(_server_tool_use_id, blocks)))
def _hosted_web_call(call_id: str, tool_name: str, tool_input: object, result: object) -> ResponseFunctionWebSearch:
return build_web_search_call(
tool_id=call_id,
tool_input=tool_input,
result=result,
status=None if result is not None else "in_progress",
tool_name=tool_name,
)
class _AnthropicUsageIteration(TypedDict, total=False):
"""One entry of the ``usage.iterations`` array on an Anthropic response."""
@ -2896,29 +2934,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
self,
web_search_results: Sequence[object],
completion_response: Mapping[str, object],
) -> list[object]:
) -> list[ResponseFunctionWebSearch]:
content: Final = completion_response.get("content")
blocks: Final = content if isinstance(content, Sequence) else ()
inputs: Final = {
call_id: tool_input
for block in blocks
if isinstance(block, Mapping)
and block.get("type") == "server_tool_use"
and block.get("name") == "web_search"
and isinstance((call_id := block.get("id")), str)
and isinstance((tool_input := block.get("input")), Mapping)
results_by_call: Final = {
(tool_use_id, result.get("type")): result
for result in web_search_results
if isinstance(result, dict) and isinstance((tool_use_id := result.get("tool_use_id")), str)
}
return [
build_web_search_call(
tool_id=tool_use_id,
tool_input=inputs.get(tool_use_id, {}),
result=result,
)
for result in web_search_results
if isinstance(result, dict)
and result.get("type") == "web_search_tool_result"
and isinstance((tool_use_id := result.get("tool_use_id")), str)
and tool_use_id in inputs
_hosted_web_call(call_id, tool_name, tool_input, results_by_call.get((call_id, f"{tool_name}_tool_result")))
for call_id, tool_name, tool_input in filter(None, map(_hosted_web_tool_use, blocks))
]
def _build_provider_specific_fields(
@ -2930,6 +2956,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
tool_results: Sequence[_AnthropicToolResultBlock] | None,
compaction_blocks: Sequence[object] | None,
tool_calls: list[ChatCompletionToolCallChunk],
server_tool_use_ids: frozenset[str],
) -> dict[str, object]:
provider_specific_fields: Final[dict[str, object]] = {
"citations": citations,
@ -2942,10 +2969,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if web_search_results is not None:
provider_specific_fields["web_search_results"] = web_search_results
provider_specific_fields["web_search_calls"] = self._build_web_search_calls(
web_search_results,
completion_response,
)
web_search_calls: Final = self._build_web_search_calls(web_search_results or (), completion_response)
if web_search_calls:
provider_specific_fields["web_search_calls"] = web_search_calls
if tool_results is not None:
provider_specific_fields["tool_results"] = tool_results
@ -2967,6 +2994,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if compaction_blocks is not None:
provider_specific_fields["compaction_blocks"] = compaction_blocks
server_tool_calls: Final = [tc for tc in tool_calls if tc["id"] in server_tool_use_ids]
if server_tool_calls:
provider_specific_fields["server_tool_calls"] = server_tool_calls
refusal_fields: Final = refusal_provider_specific_fields(
REFUSAL_STOP_FIELDS.validate_python(completion_response)
)
@ -3026,6 +3057,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
):
text_content = prefix_prompt + text_content
server_tool_use_ids: Final = _server_tool_use_ids(completion_response)
client_tool_calls: Final = [tc for tc in tool_calls if tc["id"] not in server_tool_use_ids]
provider_specific_fields: Final = self._build_provider_specific_fields(
completion_response,
citations,
@ -3034,11 +3067,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
tool_results,
compaction_blocks,
tool_calls,
server_tool_use_ids,
)
json_mode_message, tool_calls_for_message, json_extra_content = self._resolve_json_mode_non_streaming(
json_mode=json_mode,
tool_calls=tool_calls,
tool_calls=client_tool_calls,
response_format_schema=response_format_schema,
)
merged_text = text_content or ""
@ -3068,10 +3102,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
model_response.choices[0].message = _message
model_response.hidden_params["original_response"] = completion_response["content"]
model_response.choices[0].finish_reason = cast(
OpenAIChatCompletionFinishReason,
map_finish_reason(completion_response["stop_reason"]),
)
model_response.choices[0].finish_reason = map_finish_reason(completion_response["stop_reason"])
usage: Final = self.calculate_usage(
usage_object=completion_response["usage"],

198
litellm/llms/anthropic/common_utils.py
View file

@ -65,10 +65,26 @@ from litellm.types.llms.anthropic import (
AnthropicThinkingParam,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import AllMessageValues
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantToolCall,
ChatCompletionToolCallFunctionChunk,
)
from litellm.types.proxy.auth.special_headers import SpecialHeaders
from litellm.types.proxy.model_listing import ModelInfoResponse
from litellm.types.utils import LlmProviders
from litellm.types.responses.main import (
ResponseFunctionWebSearch,
web_search_call_tool_id,
with_hosted_web_tool_arguments,
)
from litellm.types.utils import (
ChatCompletionMessageCustomToolCall,
ChatCompletionMessageToolCall,
Delta,
LlmProviders,
Message,
chat_completion_tool_call_from_dict,
)
_MessageT = TypeVar("_MessageT")
@ -121,6 +137,11 @@ _DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)")
_CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
_SERVER_TOOL_CALLS_ADAPTER: Final = TypeAdapter(tuple[ChatCompletionAssistantToolCall, ...])
_SERVER_TOOL_CALL_ADAPTER: Final = TypeAdapter(ChatCompletionAssistantToolCall)
_SERVER_CALL_FIELD_KEYS: Final = ("server_tool_calls", "web_search_calls")
_WEB_SEARCH_CALL_ADAPTER: Final = TypeAdapter(ResponseFunctionWebSearch)
_TOOL_CALL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_DECODED_JSON: Final = TypeAdapter(object)
_JSON_VALUE: Final = TypeAdapter(JsonValue)
@ -2209,6 +2230,179 @@ def _replayed_server_tool_use(block: object) -> _ReplayedServerToolUse | None:
return None
def anthropic_server_tool_call_params(message: object) -> tuple[ChatCompletionAssistantToolCall, ...]:
fields: Final = validated_object_mapping(message_field(message, "provider_specific_fields"))
try:
return _SERVER_TOOL_CALLS_ADAPTER.validate_python(fields.get("server_tool_calls", ()) if fields else ())
except ValidationError:
return ()
def anthropic_server_tool_calls(message: object) -> tuple[ChatCompletionMessageToolCall, ...]:
return tuple(_message_tool_call_from_params(call) for call in anthropic_server_tool_call_params(message))
def tool_calls_with_server_tool_calls(
message: object,
) -> tuple[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall, ...]:
client_tool_calls: Final = tuple(
call
for tool_call in parts_of(message_field(message, "tool_calls"))
if (call := _client_message_tool_call(tool_call)) is not None
)
return (*anthropic_server_tool_calls(message), *client_tool_calls)
def anthropic_server_call_fields(holder: object) -> Mapping[str, object]:
fields: Final = validated_object_mapping(message_field(holder, "provider_specific_fields")) or {}
return MappingProxyType({key: fields[key] for key in _SERVER_CALL_FIELD_KEYS if fields.get(key)})
def message_dict_with_server_tool_calls(message: object) -> object:
mapping: Final = validated_object_mapping(message)
server_tool_calls: Final = anthropic_server_tool_call_params(message)
if mapping is None or not server_tool_calls:
return message
return {**mapping, "tool_calls": [*server_tool_calls, *parts_of(mapping.get("tool_calls"))]}
def _server_tool_call_param(tool_call: object) -> ChatCompletionAssistantToolCall | None:
try:
call: Final = _SERVER_TOOL_CALL_ADAPTER.validate_python(tool_call)
except ValidationError:
return None
return call if _server_tool_call_id(call).startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) else None
def message_dict_with_server_tool_calls_split_out(message: object) -> object:
mapping: Final = validated_object_mapping(message)
tool_calls: Final = parts_of(mapping.get("tool_calls")) if mapping is not None else ()
server_tool_calls: Final = tuple(
call for tool_call in tool_calls if (call := _server_tool_call_param(tool_call)) is not None
)
if mapping is None or not server_tool_calls:
return message
fields: Final = validated_object_mapping(mapping.get("provider_specific_fields")) or {}
changed: Final = changed_server_tool_call_arguments(anthropic_server_tool_call_params(message), server_tool_calls)
client_tool_calls: Final = [tool_call for tool_call in tool_calls if _server_tool_call_param(tool_call) is None]
return {
**mapping,
"tool_calls": client_tool_calls or None,
"provider_specific_fields": {
**fields,
"server_tool_calls": list(server_tool_calls),
**(
{"web_search_calls": _rewritten_web_search_calls(fields["web_search_calls"], changed)}
if changed and fields.get("web_search_calls") is not None
else {}
),
},
}
def with_server_tool_call_arguments(
call: ChatCompletionAssistantToolCall, arguments: str
) -> ChatCompletionAssistantToolCall:
return ChatCompletionAssistantToolCall(
id=call["id"],
type=call["type"],
function=ChatCompletionToolCallFunctionChunk(name=call["function"].get("name"), arguments=arguments),
)
def write_anthropic_server_tool_calls(
holder: Message | Delta, rewritten: tuple[ChatCompletionAssistantToolCall, ...]
) -> None:
original: Final = anthropic_server_tool_call_params(holder)
if rewritten == original:
return
holder.provider_specific_fields = { # rebind-ok: a guardrail rewrite must land on the message it scanned
**(holder.provider_specific_fields or {}),
"server_tool_calls": list(rewritten),
}
write_anthropic_hosted_web_call_arguments(holder, changed_server_tool_call_arguments(original, rewritten))
def _server_tool_call_id(call: ChatCompletionAssistantToolCall) -> str:
return call["id"] or ""
def _server_tool_call_arguments(call: ChatCompletionAssistantToolCall) -> str:
return call["function"].get("arguments") or ""
def changed_server_tool_call_arguments(
original: Sequence[ChatCompletionAssistantToolCall], rewritten: Sequence[ChatCompletionAssistantToolCall]
) -> Mapping[str, str]:
original_arguments: Final = MappingProxyType(
{_server_tool_call_id(call): _server_tool_call_arguments(call) for call in original}
)
return MappingProxyType(
{
_server_tool_call_id(call): _server_tool_call_arguments(call)
for call in rewritten
if _server_tool_call_arguments(call) != original_arguments.get(_server_tool_call_id(call))
}
)
def write_anthropic_hosted_web_call_arguments(holder: Message | Delta, arguments_by_tool_id: Mapping[str, str]) -> None:
fields: Final = holder.provider_specific_fields
if not arguments_by_tool_id or not fields or fields.get("web_search_calls") is None:
return
holder.provider_specific_fields = { # rebind-ok: the hosted copy of a rewritten call must not keep its old input
**fields,
"web_search_calls": _rewritten_web_search_calls(fields["web_search_calls"], arguments_by_tool_id),
}
def _rewritten_web_search_calls(calls: object, arguments_by_tool_id: Mapping[str, str]) -> object:
mapping: Final = validated_object_mapping(calls)
if mapping is not None:
return {key: _with_hosted_web_call_arguments(call, arguments_by_tool_id) for key, call in mapping.items()}
if not isinstance(calls, list):
return calls
return [_with_hosted_web_call_arguments(call, arguments_by_tool_id) for call in parts_of(calls)]
def _with_hosted_web_call_arguments(item: object, arguments_by_tool_id: Mapping[str, str]) -> object:
try:
call: Final = _WEB_SEARCH_CALL_ADAPTER.validate_python(item)
except ValidationError:
return item
arguments: Final = arguments_by_tool_id.get(web_search_call_tool_id(call.id) or "")
if arguments is None:
return item
return call.model_copy(update={"action": with_hosted_web_tool_arguments(call.action, arguments)})
def write_anthropic_server_tool_call_arguments(holder: Message | Delta, index: int, arguments: str) -> None:
write_anthropic_server_tool_calls(
holder,
tuple(
with_server_tool_call_arguments(call, arguments) if call_index == index else call
for call_index, call in enumerate(anthropic_server_tool_call_params(holder))
),
)
def _message_tool_call_from_params(call: ChatCompletionAssistantToolCall) -> ChatCompletionMessageToolCall:
return ChatCompletionMessageToolCall(id=call["id"], type=call["type"], function=dict(call["function"]))
def _client_message_tool_call(
tool_call: object,
) -> ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall | None:
match tool_call:
case ChatCompletionMessageToolCall() | ChatCompletionMessageCustomToolCall():
return tool_call
case _:
try:
return chat_completion_tool_call_from_dict(_TOOL_CALL_FIELDS_ADAPTER.validate_python(tool_call))
except (TypeError, ValidationError):
return None
def tool_call_is_rebuilt_as_server_tool_use(
tool_call_id: object, provider_specific_fields: object, answered_tool_ids: Container[str] = frozenset()
) -> bool:

22
litellm/llms/anthropic/messages/adapters/transformation.py
View file

@ -46,6 +46,25 @@ def _thought_signature(provider_specific_fields: object) -> str | None:
return signature if isinstance(signature, str) else None
def _object_sequence(value: object) -> Sequence[object] | None:
return value if isinstance(value, (list, tuple)) else None
def _server_tool_blocks(message: object) -> tuple[Mapping[str, object], ...]:
server_tool_calls: Final = anthropic_server_tool_call_params(message)
if not server_tool_calls:
return ()
fields: Final = _as_string_mapping(_optional_attr(message, "provider_specific_fields")) or {}
return tuple(
convert_to_anthropic_tool_invoke(
server_tool_calls,
web_search_results=_object_sequence(fields.get("web_search_results")),
tool_results=_object_sequence(fields.get("tool_results")),
server_tool_call_ids=frozenset(tool_call["id"] for tool_call in server_tool_calls),
)
)
def _compaction_blocks(provider_specific_fields: object) -> tuple[Mapping[str, object], ...]:
fields: Final = _as_string_mapping(provider_specific_fields)
raw_blocks: Final = fields.get("compaction_blocks") if fields is not None else None
@ -131,12 +150,14 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
)
from litellm.litellm_core_utils.prompt_templates.factory import (
THOUGHT_SIGNATURE_SEPARATOR,
convert_to_anthropic_tool_invoke,
)
from litellm.litellm_core_utils.reasoning_effort_utils import (
reasoning_effort_from_thinking,
reasoning_effort_from_thinking_silently,
)
from litellm.llms.anthropic.common_utils import (
anthropic_server_tool_call_params,
eager_input_streaming_flag,
is_empty_unsigned_thinking_block,
normalize_anthropic_tool_use_id,
@ -1462,6 +1483,7 @@ class LiteLLMAnthropicMessagesAdapter:
).model_dump()
)
new_content.extend(dict(block) for block in _server_tool_blocks(choice.message))
# Handle text content
if choice.message.content is not None and (choice.message.content != "" or not compaction_blocks):
new_content.append(

26
litellm/llms/base_llm/guardrail_translation/utils.py
View file

@ -4,11 +4,13 @@ import json
from collections.abc import Callable, Iterator, Mapping, Sequence
from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row has no typed constructor across roles
from openai.types.responses import ResponseFunctionWebSearch
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage
from litellm.types.responses.main import with_hosted_web_tool_arguments
_DECODED_JSON: Final = TypeAdapter(object)
@ -135,6 +137,30 @@ def stream_item_field(item: object, field: str) -> object | None:
return getattr(item, field, None)
def web_search_call(item: object) -> ResponseFunctionWebSearch | None:
if isinstance(item, ResponseFunctionWebSearch):
return item
if stream_item_field(item, "type") != "web_search_call":
return None
try:
return ResponseFunctionWebSearch.model_validate(item.model_dump() if isinstance(item, BaseModel) else item)
except ValidationError:
return None
def with_web_search_call_arguments(item: object, arguments: str) -> object:
search: Final = web_search_call(item)
if search is None:
return item
action: Final = with_hosted_web_tool_arguments(search.action, arguments)
if isinstance(item, ResponseFunctionWebSearch):
return item.model_copy(update={"action": action})
if isinstance(item, BaseModel):
return item.model_copy(update={"action": action.model_dump()})
fields: Final = cast("Mapping[str, object]", item) # cast-ok: a validated web_search_call dict is str-keyed JSON
return {**fields, "action": action.model_dump()}
def stream_item_fingerprint(item: object) -> str:
plain: Final = item.model_dump() if isinstance(item, BaseModel) else item
return json.dumps(plain, sort_keys=True, default=str)

147
litellm/llms/openai/chat/guardrail_translation/handler.py
View file

@ -21,6 +21,7 @@ from collections.abc import Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Union, cast
from pydantic import TypeAdapter, ValidationError
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
@ -29,6 +30,12 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i
replayed_tool_search_tool_names,
)
from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder
from litellm.llms.anthropic.common_utils import (
anthropic_server_tool_call_params,
changed_server_tool_call_arguments,
write_anthropic_hosted_web_call_arguments,
write_anthropic_server_tool_calls,
)
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
StreamingScanKey,
@ -50,7 +57,12 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
stream_item_items,
unappliable_request_rewrite,
)
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantToolCall,
ChatCompletionToolCallFunctionChunk,
ChatCompletionToolParam,
)
from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
coerce_stream_holdback_value,
)
@ -436,9 +448,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
text_task_mappings=text_task_mappings,
tool_call_task_mappings=tool_call_task_mappings,
)
server_tool_call_tasks: Final = tuple(_server_tool_call_tasks(response.choices))
tool_calls_with_server_calls: Final = [
*tool_calls_to_check,
*(_server_tool_call_input(call) for _, call in server_tool_call_tasks),
]
# Step 2: Apply guardrail to all texts and tool calls in batch
if texts_to_check or tool_calls_to_check:
if texts_to_check or tool_calls_with_server_calls:
# Use the real request_data if provided (proxy path), otherwise
# create a standalone dict (SDK / direct-call path).
if request_data is None:
@ -452,8 +469,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
if images_to_check:
inputs["images"] = images_to_check
if tool_calls_to_check:
inputs["tool_calls"] = tool_calls_to_check
if tool_calls_with_server_calls:
inputs["tool_calls"] = tool_calls_with_server_calls
# Include model information from the response if available
if hasattr(response, "model") and response.model:
inputs["model"] = response.model
@ -469,8 +486,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
returned_tool_calls: Final = guardrailed_inputs.get("tool_calls")
guardrailed_tool_calls: Final[list[dict[str, object]]] = (
cast(list[dict[str, object]], returned_tool_calls)
if isinstance(returned_tool_calls, list) and len(returned_tool_calls) == len(tool_calls_to_check)
else tool_calls_to_check
if isinstance(returned_tool_calls, list)
and len(returned_tool_calls) == len(tool_calls_with_server_calls)
else tool_calls_with_server_calls
)
# Step 3: Map guardrail responses back to original response structure
@ -488,6 +506,11 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
tool_calls=guardrailed_tool_calls,
task_mappings=tool_call_task_mappings,
)
self._apply_guardrail_responses_to_output_server_tool_calls(
response=response,
guardrailed_tool_calls=guardrailed_tool_calls[len(tool_calls_to_check) :],
task_mappings=tuple(mapping for mapping, _ in server_tool_call_tasks),
)
verbose_proxy_logger.debug("OpenAI Chat Completions: Processed output response: %s", response)
@ -681,6 +704,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
pre_guardrail_tool_calls=pre_guardrail_tool_calls,
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
)
self._write_ended_stream_server_tool_call_rewrites(
responses_so_far=responses_so_far,
guardrailed_response=model_response,
)
@staticmethod
def _rebuild_ended_stream_per_choice(
@ -934,6 +961,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if choice.message.tool_calls and isinstance(choice.message.tool_calls, list):
if len(choice.message.tool_calls) > 0:
return True
if anthropic_server_tool_call_params(choice.message):
return True
elif isinstance(response, ModelResponseStream):
for streaming_choice in response.choices:
if isinstance(streaming_choice, litellm.StreamingChoices):
@ -944,6 +973,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if streaming_choice.delta.tool_calls and isinstance(streaming_choice.delta.tool_calls, list):
if len(streaming_choice.delta.tool_calls) > 0:
return True
if anthropic_server_tool_call_params(streaming_choice.delta):
return True
return False
def _extract_output_text_images_and_tool_calls(
@ -1094,6 +1125,22 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if "name" in func_dict:
existing_tool_call.function.name = func_dict["name"]
@staticmethod
def _apply_guardrail_responses_to_output_server_tool_calls(
response: "ModelResponse",
guardrailed_tool_calls: Sequence[object],
task_mappings: Sequence[tuple[int, int]],
) -> None:
guardrailed_by_task: Final = MappingProxyType(dict(zip(task_mappings, guardrailed_tool_calls)))
for choice_idx, choice in enumerate(response.choices):
write_anthropic_server_tool_calls(
choice.message,
tuple(
_rewritten_server_tool_call(server_tool_call, guardrailed_by_task.get((choice_idx, server_idx)))
for server_idx, server_tool_call in enumerate(anthropic_server_tool_call_params(choice.message))
),
)
@staticmethod
def _string_choice_contents(response: "ModelResponse") -> tuple[str | None, ...]:
return tuple(
@ -1206,6 +1253,42 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
for fragment in tail:
fragment.function.arguments = ""
@staticmethod
def _write_ended_stream_server_tool_call_rewrites(
responses_so_far: Sequence["ModelResponseStream"],
guardrailed_response: "ModelResponse",
) -> None:
guardrailed_by_id: Final = MappingProxyType(
{
(guardrailed_response.choices[choice_position].index, server_tool_call["id"]): server_tool_call
for (choice_position, _), server_tool_call in _server_tool_call_tasks(guardrailed_response.choices)
}
)
if not guardrailed_by_id:
return
original_by_id: Final = MappingProxyType(dict(_streamed_server_tool_calls(responses_so_far)))
changed_by_choice: Final = MappingProxyType(
{
choice_index: changed_server_tool_call_arguments(
_choice_server_tool_calls(original_by_id, choice_index),
_choice_server_tool_calls(guardrailed_by_id, choice_index),
)
for choice_index in frozenset(index for index, _ in guardrailed_by_id)
}
)
for response in responses_so_far:
for choice in response.choices:
write_anthropic_server_tool_calls(
choice.delta,
tuple(
guardrailed_by_id.get((choice.index, server_tool_call["id"]), server_tool_call)
for server_tool_call in anthropic_server_tool_call_params(choice.delta)
),
)
write_anthropic_hosted_web_call_arguments(
choice.delta, changed_by_choice.get(choice.index, MappingProxyType({}))
)
async def _apply_guardrail_responses_to_output_streaming(
self,
responses: list["ModelResponseStream"],
@ -1395,7 +1478,57 @@ def _stream_chunk_choices(item: object) -> Sequence[object]:
def _streamed_delta_tool_calls(delta: object) -> tuple[object, ...]:
function_call: Final = stream_item_field(delta, "function_call")
legacy: Final = () if function_call is None else (function_call,)
return stream_item_items(delta, "tool_calls") + legacy
return anthropic_server_tool_call_params(delta) + stream_item_items(delta, "tool_calls") + legacy
def _server_tool_call_tasks(
choices: Sequence[object],
) -> Iterator[tuple[tuple[int, int], ChatCompletionAssistantToolCall]]:
for choice_idx, choice in enumerate(choices):
if not isinstance(choice, Choices):
continue
for server_idx, server_tool_call in enumerate(anthropic_server_tool_call_params(choice.message)):
yield (choice_idx, server_idx), server_tool_call
def _streamed_server_tool_calls(
responses: Sequence["ModelResponseStream"],
) -> Iterator[tuple[tuple[int, str | None], ChatCompletionAssistantToolCall]]:
for response in responses:
for choice in response.choices:
for server_tool_call in anthropic_server_tool_call_params(choice.delta):
yield (choice.index, server_tool_call["id"]), server_tool_call
def _choice_server_tool_calls(
calls_by_id: Mapping[tuple[int, str | None], ChatCompletionAssistantToolCall], choice_index: int
) -> tuple[ChatCompletionAssistantToolCall, ...]:
return tuple(call for (index, _), call in calls_by_id.items() if index == choice_index)
def _server_tool_call_input(server_tool_call: ChatCompletionAssistantToolCall) -> dict[str, object]:
return {
"id": server_tool_call["id"],
"type": server_tool_call["type"],
"function": dict(server_tool_call["function"]),
}
_TOOL_CALL_FUNCTION_ADAPTER: Final = TypeAdapter(ChatCompletionToolCallFunctionChunk)
def _rewritten_server_tool_call(
server_tool_call: ChatCompletionAssistantToolCall, guardrailed: object
) -> ChatCompletionAssistantToolCall:
try:
function: Final = _TOOL_CALL_FUNCTION_ADAPTER.validate_python(stream_item_field(guardrailed, "function"))
except ValidationError:
return server_tool_call
return ChatCompletionAssistantToolCall(
id=server_tool_call["id"],
type=server_tool_call["type"],
function={**server_tool_call["function"], **function},
)
def _released_choices(responses_so_far: Sequence[object]) -> Iterator[object]:

71
litellm/llms/openai/responses/guardrail_translation/handler.py
View file

@ -66,6 +66,8 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
stream_item_fingerprint,
stream_item_items,
unappliable_request_rewrite,
web_search_call,
with_web_search_call_arguments,
)
from litellm.llms.openai.responses.guardrail_translation.tool_merge import (
append_returned_tools,
@ -81,12 +83,14 @@ from litellm.responses.litellm_completion_transformation.tool_search import is_s
from litellm.responses.litellm_completion_transformation.transformation import (
TOOL_CALL_OUTPUT_ITEM_TYPES,
LiteLLMCompletionResponsesConfig,
claude_web_search_call_tool_call,
tool_output_to_chat_content,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import (
AllMessageValues,
BaseLiteLLMOpenAIResponseObject,
ChatCompletionAssistantToolCall,
ChatCompletionToolCallChunk,
ChatCompletionToolParam,
ContentPartAddedEvent,
@ -260,6 +264,8 @@ _TERMINAL_ENVELOPE_EVENT_TYPES: Final = frozenset(
_TOOL_CALL_ITEM_TYPES: Final = frozenset({"function_call", "custom_tool_call"})
_TOOL_SEARCH_CALL_TYPE: Final = "tool_search_call"
_WEB_SEARCH_CALL_TYPE: Final = "web_search_call"
_ITEM_ID_KEYED_TOOL_CALL_TYPES: Final = frozenset({_TOOL_SEARCH_CALL_TYPE, _WEB_SEARCH_CALL_TYPE})
_OUTPUT_TOOL_CALL_ITEM_TYPES: Final = _TOOL_CALL_ITEM_TYPES | {_TOOL_SEARCH_CALL_TYPE}
_SCANNED_TOOL_SEARCH_NAME: Final = "tool_search"
_TOOL_SEARCH_ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object])
@ -328,8 +334,27 @@ def _is_tool_call_item(item: object) -> bool:
return isinstance(item, Mapping) and item.get("type") in _TOOL_CALL_ITEM_TYPES
def _claude_web_search_tool_call(item: object) -> ChatCompletionAssistantToolCall | None:
search: Final = web_search_call(item)
return None if search is None else claude_web_search_call_tool_call(search)
def _claude_web_tool_call(item: object, index: int) -> ChatCompletionToolCallChunk | None:
tool_call: Final = _claude_web_search_tool_call(item)
if tool_call is None:
return None
return ChatCompletionToolCallChunk(id=tool_call["id"], type="function", function=tool_call["function"], index=index)
def _is_output_tool_call_item(item: object) -> bool:
return (
stream_item_field(item, "type") in _OUTPUT_TOOL_CALL_ITEM_TYPES
or _claude_web_search_tool_call(item) is not None
)
def _tool_call_output_item_mapping(item: object) -> Mapping[str, object] | None:
if stream_item_field(item, "type") not in _OUTPUT_TOOL_CALL_ITEM_TYPES:
if not _is_output_tool_call_item(item):
return None
if isinstance(item, Mapping):
return cast("Mapping[str, object]", item) # cast-ok: output items are str-keyed JSON objects
@ -341,7 +366,7 @@ def _tool_call_item_key(item: object) -> str | None:
if isinstance(call_id, str) and call_id:
return call_id
item_id: Final = stream_item_field(item, "id")
if stream_item_field(item, "type") == _TOOL_SEARCH_CALL_TYPE and isinstance(item_id, str) and item_id:
if stream_item_field(item, "type") in _ITEM_ID_KEYED_TOOL_CALL_TYPES and isinstance(item_id, str) and item_id:
return item_id
return None
@ -361,6 +386,17 @@ def _scanned_tool_call_item(item: Mapping[str, object]) -> Mapping[str, object]:
return {**item, "name": _SCANNED_TOOL_SEARCH_NAME, "arguments": json.dumps(item.get("arguments") or {})}
def _scanned_tool_call(item: Mapping[str, object], index: int) -> ChatCompletionToolCallChunk:
web_tool_call: Final = _claude_web_tool_call(item, index)
if web_tool_call is not None:
return web_tool_call
return tool_call_dict_from_output_item(_scanned_tool_call_item(item), index)
def _rewritten_web_search_call(item: object, payload: str | None) -> object:
return item if payload is None else with_web_search_call_arguments(item, payload)
def _tool_search_rewrite(payload: str) -> Mapping[str, object]:
try:
return {"arguments": _TOOL_SEARCH_ARGUMENTS_ADAPTER.validate_json(payload)}
@ -1299,17 +1335,22 @@ class OpenAIResponsesHandler(BaseTranslation):
# delivered, so a truncated buffer still reports text undeliverable. #
# ------------------------------------------------------------------ #
if final_chunk.get("type") == "response.output_item.done":
finished_summary: Final = _output_compaction_summary(stream_item_field(final_chunk, "item"))
finished_item: Final = stream_item_field(final_chunk, "item")
finished_summary: Final = _output_compaction_summary(finished_item)
model_response_stream: Final = (
OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(final_chunk)
)
tool_calls: Final = model_response_stream.choices[0].delta.tool_calls
finished_web_tool_call: Final = _claude_web_tool_call(finished_item, 0)
tool_calls: Final = [
*cast(list[ChatCompletionToolCallChunk], model_response_stream.choices[0].delta.tool_calls or []),
*(() if finished_web_tool_call is None else (finished_web_tool_call,)),
]
if tool_calls or finished_summary is not None:
inputs = GenericGuardrailAPIInputs()
if finished_summary is not None:
inputs["texts"] = [finished_summary]
if tool_calls:
inputs["tool_calls"] = cast(list[ChatCompletionToolCallChunk], tool_calls)
inputs["tool_calls"] = tool_calls
if hasattr(model_response_stream, "model") and model_response_stream.model:
inputs["model"] = model_response_stream.model
await guardrail_to_apply.apply_guardrail(
@ -1598,7 +1639,7 @@ class OpenAIResponsesHandler(BaseTranslation):
if event_type not in _OUTPUT_ITEM_EVENT_TYPES:
return None
item: Final = stream_item_field(event, "item")
return _tool_call_item_key(item) if stream_item_field(item, "type") in _OUTPUT_TOOL_CALL_ITEM_TYPES else None
return _tool_call_item_key(item) if _is_output_tool_call_item(item) else None
@staticmethod
def _write_tool_call_item(item: object, name: str | None, payload: str | None) -> object:
@ -1612,6 +1653,8 @@ class OpenAIResponsesHandler(BaseTranslation):
return item
if item_type == _TOOL_SEARCH_CALL_TYPE:
return _rewritten_tool_search_call(item, payload)
if item_type == _WEB_SEARCH_CALL_TYPE:
return _rewritten_web_search_call(item, payload)
if name is not None:
OpenAIResponsesHandler._write_event_field(item, "name", name)
if payload is not None and isinstance(item_type, str) and item_type in _TOOL_CALL_PAYLOAD_FIELDS:
@ -1667,7 +1710,7 @@ class OpenAIResponsesHandler(BaseTranslation):
stream_item_field(event, "type") in _TOOL_CALL_PAYLOAD_EVENT_TYPES
or (
stream_item_field(event, "type") in _OUTPUT_ITEM_EVENT_TYPES
and stream_item_field(stream_item_field(event, "item"), "type") in _OUTPUT_TOOL_CALL_ITEM_TYPES
and _is_output_tool_call_item(stream_item_field(event, "item"))
)
for event in responses_so_far
)
@ -1675,9 +1718,7 @@ class OpenAIResponsesHandler(BaseTranslation):
@staticmethod
def _completed_response_scan_key(response: object) -> StreamingScanKey:
output_items: Final = stream_item_items(response, "output")
message_items: Final = tuple(
item for item in output_items if stream_item_field(item, "type") not in _OUTPUT_TOOL_CALL_ITEM_TYPES
)
message_items: Final = tuple(item for item in output_items if not _is_output_tool_call_item(item))
message_parts: Final = chain.from_iterable(stream_item_items(item, "content") for item in message_items)
part_texts: Final = tuple(
text for part in message_parts if isinstance(text := stream_item_field(part, "text"), str) and text
@ -1685,11 +1726,7 @@ class OpenAIResponsesHandler(BaseTranslation):
summaries: Final = tuple(summary for item in output_items if (summary := _output_compaction_summary(item)))
return StreamingScanKey(
texts=(*part_texts, *summaries),
tool_calls=tuple(
stream_item_fingerprint(item)
for item in output_items
if stream_item_field(item, "type") in _OUTPUT_TOOL_CALL_ITEM_TYPES
),
tool_calls=tuple(stream_item_fingerprint(item) for item in output_items if _is_output_tool_call_item(item)),
stream_ended=True,
)
@ -1802,9 +1839,7 @@ class OpenAIResponsesHandler(BaseTranslation):
tool_call_item: Final = _tool_call_output_item_mapping(output_item)
if tool_call_item is not None:
if tool_calls_to_check is not None:
tool_calls_to_check.append(
tool_call_dict_from_output_item(_scanned_tool_call_item(tool_call_item), output_idx)
)
tool_calls_to_check.append(_scanned_tool_call(tool_call_item, output_idx))
return
summary: Final = _output_compaction_summary(output_item)

17
litellm/proxy/db/spend_log_tool_index.py
View file

@ -63,11 +63,26 @@ def response_tool_call_names(completion_response: object) -> tuple[str, ...]:
)
return tuple(
stripped
for name in chain(called_names, _auto_executed_mcp_tool_names(completion_response))
for name in chain(
_server_tool_call_names(completion_response),
called_names,
_auto_executed_mcp_tool_names(completion_response),
)
if isinstance(name, str) and (stripped := name.strip())
)
def _server_tool_call_names(completion_response: object) -> tuple[str | None, ...]:
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
server_tool_calls: Final = chain.from_iterable(
anthropic_server_tool_call_params(message_field(choice, "message"))
for choice in parts_of(message_field(completion_response, "choices"))
)
return tuple(server_tool_call["function"].get("name") for server_tool_call in server_tool_calls)
class _ExecutedTool(BaseModel):
model_config = ConfigDict(frozen=True)

50
litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py
View file

@ -24,6 +24,10 @@ from litellm._logging import verbose_proxy_logger
from litellm._version import version as litellm_version
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.litellm_core_utils.core_helpers import iter_items
from litellm.llms.anthropic.common_utils import (
anthropic_server_tool_calls,
write_anthropic_server_tool_call_arguments,
)
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
get_ssl_configuration,
@ -515,27 +519,37 @@ class CatoNetworksGuardrail(CustomGuardrail):
@staticmethod
def _output_fragments(message: Message) -> Sequence[tuple[tuple[str, int | None], str]]:
"""Assistant text the proxy returns to the caller: ``content`` plus every
``tool_calls[].function.arguments`` string, each tagged with where a
redaction must be written back. ``content`` is only included when present
so a tool-call-only choice keeps its ``None`` content (the text-vs-tool-call
signal downstream consumers rely on) while its arguments are still inspected."""
fragments: Final[list] = []
if message.content is not None:
fragments.append((("content", None), message.content))
for idx, tool_call in enumerate(message.tool_calls or []):
function = getattr(tool_call, "function", None)
arguments = getattr(function, "arguments", None)
if isinstance(arguments, str) and arguments:
fragments.append((("tool_call", idx), arguments))
return fragments
tool call's ``function.arguments`` string, Claude's server tool calls first,
each tagged with where a redaction must be written back. ``content`` is only
included when present so a tool-call-only choice keeps its ``None`` content
(the text-vs-tool-call signal downstream consumers rely on) while its
arguments are still inspected."""
content: Final = () if message.content is None else ((("content", None), message.content),)
return (
*content,
*CatoNetworksGuardrail._tool_call_fragments("server_tool_call", anthropic_server_tool_calls(message)),
*CatoNetworksGuardrail._tool_call_fragments("tool_call", message.tool_calls or ()),
)
@staticmethod
def _tool_call_fragments(kind: str, tool_calls: Sequence[object]) -> tuple[tuple[tuple[str, int | None], str], ...]:
return tuple(
((kind, idx), arguments)
for idx, arguments in enumerate(
getattr(getattr(tool_call, "function", None), "arguments", None) for tool_call in tool_calls
)
if isinstance(arguments, str) and arguments
)
@staticmethod
def _apply_output_fragment(message: Any, target: tuple[str, int | None], redacted: str) -> None:
kind, idx = target
if kind == "content":
message.content = redacted
else:
message.tool_calls[idx].function.arguments = redacted
match target:
case ("content", _):
message.content = redacted
case ("server_tool_call", int() as idx):
write_anthropic_server_tool_call_arguments(message, idx, redacted)
case (_, idx):
message.tool_calls[idx].function.arguments = redacted
@staticmethod
def _responses_output_field(item: object, key: str) -> str | Sequence[object] | None:

52
litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py
View file

@ -37,6 +37,16 @@ from litellm.integrations.custom_guardrail import (
log_guardrail_information,
)
from litellm.litellm_core_utils.core_helpers import iter_items
from litellm.llms.anthropic.common_utils import (
anthropic_server_tool_call_params,
tool_calls_with_server_tool_calls,
with_server_tool_call_arguments,
write_anthropic_server_tool_calls,
)
from litellm.llms.base_llm.guardrail_translation.utils import (
web_search_call,
with_web_search_call_arguments,
)
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
@ -46,9 +56,12 @@ from litellm.proxy.common_utils.callback_utils import (
add_guardrail_to_applied_guardrails_header,
)
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.llms.anthropic import claude_hosted_web_tool_call
from litellm.types.utils import (
Choices,
Delta,
LLMResponseTypes,
Message,
ModelResponse,
ModelResponseStream,
TextCompletionResponse,
@ -121,6 +134,12 @@ class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object):
"""Base-class constructor options this guardrail forwards untouched to CustomGuardrail."""
def _claude_hosted_web_tool_arguments(item: object) -> str | None:
search: Final = web_search_call(item)
hosted_call: Final = None if search is None else claude_hosted_web_tool_call(search)
return None if hosted_call is None else hosted_call[2]
def _streamed_choice_index(choice: object) -> int:
index: Final[object] = getattr(choice, "index", None)
return index if isinstance(index, int) else 0
@ -1654,6 +1673,7 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail):
@classmethod
def _clear_tool_call_arguments(cls, message: object) -> None:
"""Clear tool-call / function-call arguments after Cisco redaction."""
cls._clear_server_tool_call_arguments(message)
tool_calls = message.get("tool_calls") if isinstance(message, dict) else getattr(message, "tool_calls", None)
for tc in tool_calls or []:
fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None)
@ -1663,9 +1683,26 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail):
)
cls._clear_arguments_field(function_call)
@staticmethod
def _clear_server_tool_call_arguments(message: object) -> None:
cleared: Final = tuple(
with_server_tool_call_arguments(server_tool_call, "{}")
for server_tool_call in anthropic_server_tool_call_params(message)
)
match message:
case Message() | Delta():
write_anthropic_server_tool_calls(message, cleared)
case dict() if cleared:
message["provider_specific_fields"] = { # rebind-ok: a redaction must land on the message it scanned
**(message.get("provider_specific_fields") or {}),
"server_tool_calls": list(cleared),
}
case _:
pass
def _redact_responses_api_output(
self,
output_items: list,
output_items: list[object],
sanitized_text: str | None,
sanitized_messages: list[dict[str, object]] | None,
) -> bool:
@ -1677,7 +1714,10 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail):
if not replacement_text:
return False
applied = False
for item in output_items:
for index, item in enumerate(output_items):
if _claude_hosted_web_tool_arguments(item):
output_items[index] = with_web_search_call_arguments(item, "{}") # rebind-ok: in-place redaction
applied = True
content = getattr(item, "content", None) or (item.get("content") if isinstance(item, dict) else None)
if isinstance(content, list):
for part in content:
@ -2038,8 +2078,9 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail):
output_items: Final = getattr(response, "output", None)
if not isinstance(output_items, list):
return []
typed_output_items: Final[Sequence[object]] = output_items
output_parts: Final[list[str]] = []
for item in output_items:
for item in typed_output_items:
get = item.get if isinstance(item, dict) else (lambda k: getattr(item, k, None))
for part in get("content") or []:
pget = part.get if isinstance(part, dict) else (lambda k: getattr(part, k, None))
@ -2053,6 +2094,8 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail):
direct = get("text")
if isinstance(direct, str) and direct:
output_parts.append(direct)
if hosted_arguments := _claude_hosted_web_tool_arguments(item):
output_parts.append(hosted_arguments)
joined = " ".join(output_parts)
return [{"role": "assistant", "content": joined}] if joined else []
@ -2095,8 +2138,7 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail):
@classmethod
def _extract_message_tool_argument_parts(cls, message: object) -> list[str]:
parts: Final[list[str]] = []
tool_calls = message.get("tool_calls") if isinstance(message, dict) else getattr(message, "tool_calls", None)
for tool_call in tool_calls or []:
for tool_call in tool_calls_with_server_tool_calls(message):
args = cls._extract_tool_call_arguments(tool_call)
if args:
parts.append(args)

27
litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py
View file

@ -40,6 +40,12 @@ from litellm.integrations.custom_guardrail import (
)
from litellm.integrations.custom_guardrail import dc as global_cache
from litellm.litellm_core_utils.core_helpers import iter_items
from litellm.llms.anthropic.common_utils import (
anthropic_server_tool_call_params,
anthropic_server_tool_calls,
with_server_tool_call_arguments,
write_anthropic_server_tool_calls,
)
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
@ -50,6 +56,7 @@ from litellm.proxy.guardrails._content_utils import (
has_non_string_content,
)
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.llms.openai import ChatCompletionAssistantToolCall
class LassoResponse(TypedDict):
@ -259,7 +266,7 @@ class LassoGuardrail(CustomGuardrail):
msg = choice.message
if msg.content:
response_messages.append({"role": "assistant", "content": msg.content})
for call in getattr(msg, "tool_calls", None) or []:
for call in (*anthropic_server_tool_calls(msg), *(getattr(msg, "tool_calls", None) or ())):
call_id, name, input_data = self._extract_tool_call_fields(call)
if not call_id or not name:
continue
@ -962,6 +969,14 @@ class LassoGuardrail(CustomGuardrail):
text_cursor += 1
verbose_proxy_logger.debug("Applied masked text content to choice %s", text_cursor)
write_anthropic_server_tool_calls(
msg,
tuple(
self._masked_server_tool_call(server_tool_call, masked_tool_use)
for server_tool_call in anthropic_server_tool_call_params(msg)
),
)
for call in getattr(msg, "tool_calls", None) or []:
call_id = self._get_field(call, "id")
if call_id and call_id in masked_tool_use:
@ -977,6 +992,16 @@ class LassoGuardrail(CustomGuardrail):
func.arguments = json.dumps(masked_input)
verbose_proxy_logger.debug("Applied masked tool_call arguments for call_id=%s", call_id)
@staticmethod
def _masked_server_tool_call(
server_tool_call: ChatCompletionAssistantToolCall,
masked_tool_use: Mapping[object, Mapping[str, object]],
) -> ChatCompletionAssistantToolCall:
masked_input: Final = masked_tool_use.get(server_tool_call["id"], {}).get("input")
if masked_input is None:
return server_tool_call
return with_server_tool_call_arguments(server_tool_call, json.dumps(masked_input))
@staticmethod
def get_config_model() -> type["GuardrailConfigModel"] | None:
from litellm.types.proxy.guardrails.guardrail_hooks.lasso import (

20
litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py
View file

@ -24,13 +24,17 @@ from litellm.integrations.custom_guardrail import (
log_guardrail_information,
)
from litellm.litellm_core_utils.core_helpers import attribute_items, attribute_or_none, iter_items, require_attributes
from litellm.llms.anthropic.common_utils import (
anthropic_server_tool_call_params,
write_anthropic_server_tool_call_arguments,
)
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.utils import GenericGuardrailAPIInputs, TextChoices
from litellm.types.utils import GenericGuardrailAPIInputs, Message, TextChoices
if TYPE_CHECKING:
from litellm.caching.caching import DualCache
@ -87,6 +91,19 @@ _DEFAULT_TIMEOUT_SECONDS: Final = 10.0
_SHIELD_BODY: Final = TypeAdapter(dict[str, object])
def _server_tool_call_slots(message: object) -> tuple[Slot, ...]:
if not isinstance(message, Message):
return ()
return tuple(
(arguments, functools.partial(write_anthropic_server_tool_call_arguments, message, index))
for index, arguments in enumerate(
server_tool_call["function"].get("arguments")
for server_tool_call in anthropic_server_tool_call_params(message)
)
if isinstance(arguments, str) and arguments
)
def _string_tool_arguments(calls: Sequence[object]) -> Iterator[tuple[object, str]]:
for call in calls:
function = read_field(call, "function")
@ -382,6 +399,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail):
content = getattr(message, "content", None)
if isinstance(content, str) and content:
pending.append((content, functools.partial(setattr, message, "content")))
pending.extend(_server_tool_call_slots(message))
for tool_call in attribute_items(message, "tool_calls"):
function: object = getattr(tool_call, "function", None)
arguments = getattr(function, "arguments", None) if function is not None else None

5
litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/base.py
View file

@ -13,6 +13,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
)
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
@ -434,9 +435,7 @@ class PurviewGuardrailBase:
args: Final[list[str]] = []
# tool_calls: [{"function": {"arguments": "..."}}]
tool_calls: Final[Sequence[object] | None] = (
message.get("tool_calls") if isinstance(message, dict) else getattr(message, "tool_calls", None)
)
tool_calls: Final = tool_calls_with_server_tool_calls(message)
if tool_calls:
for tc in tool_calls:
fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None)

25
litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py
View file

@ -1,5 +1,6 @@
# litellm/proxy/guardrails/guardrail_hooks/pangea.py
import os
from collections.abc import Callable
from typing import TYPE_CHECKING, Final
from fastapi import HTTPException
@ -10,6 +11,11 @@ from litellm.integrations.custom_guardrail import (
CustomGuardrail,
log_guardrail_information,
)
from litellm.llms.anthropic.common_utils import (
message_dict_with_server_tool_calls,
message_dict_with_server_tool_calls_split_out,
validated_object_mapping,
)
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
@ -30,6 +36,13 @@ if TYPE_CHECKING:
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
def _choice_with_message(choice: object, rewrite: Callable[[object], object]) -> object:
mapping: Final = validated_object_mapping(choice)
if mapping is None or "message" not in mapping:
return choice
return {**mapping, "message": rewrite(mapping["message"])}
class PangeaGuardrailMissingSecrets(Exception):
"""Custom exception for missing Pangea secrets."""
@ -242,9 +255,13 @@ class PangeaHandler(CustomGuardrail):
for c in choices:
if isinstance(c, Choices):
try:
serialized_choices.append(c.model_dump())
serialized_choices.append(
_choice_with_message(c.model_dump(), message_dict_with_server_tool_calls)
)
except Exception:
serialized_choices.append(c.dict())
serialized_choices.append(
_choice_with_message(c.dict(), message_dict_with_server_tool_calls)
)
else:
serialized_choices.append(c)
choices = serialized_choices
@ -269,7 +286,9 @@ class PangeaHandler(CustomGuardrail):
return
output: Final = ai_guard_response.get("result", {}).get("output", {})
response.choices = output["choices"]
response.choices = [ # pyright: ignore[reportAttributeAccessIssue] # Pangea returns untyped choice dicts
_choice_with_message(choice, message_dict_with_server_tool_calls_split_out) for choice in output["choices"]
]
return response
@log_guardrail_information

21
litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py
View file

@ -27,6 +27,12 @@ from litellm.integrations.custom_guardrail import (
CustomGuardrail,
log_guardrail_information,
)
from litellm.llms.anthropic.common_utils import (
anthropic_server_tool_call_params,
tool_calls_with_server_tool_calls,
with_server_tool_call_arguments,
write_anthropic_server_tool_calls,
)
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_scan_only_tool_results_for_guardrail,
effective_skip_system_message_for_guardrail,
@ -364,10 +370,9 @@ class PanwPrismaAirsHandler(CustomGuardrail):
text_parts.append(str(choice.message.content))
# Extract tool call arguments
if hasattr(choice.message, "tool_calls") and choice.message.tool_calls:
for tool_call in choice.message.tool_calls:
if hasattr(tool_call, "function") and hasattr(tool_call.function, "arguments"):
text_parts.append(str(tool_call.function.arguments))
for tool_call in tool_calls_with_server_tool_calls(choice.message):
if hasattr(tool_call, "function") and hasattr(tool_call.function, "arguments"):
text_parts.append(str(tool_call.function.arguments))
# Extract function call arguments (legacy)
if hasattr(choice.message, "function_call") and choice.message.function_call:
@ -756,6 +761,14 @@ class PanwPrismaAirsHandler(CustomGuardrail):
elif isinstance(content, list):
choice.message.content = self._mask_content_list(content, masked_text)
write_anthropic_server_tool_calls(
choice.message,
tuple(
with_server_tool_call_arguments(server_tool_call, masked_text)
for server_tool_call in anthropic_server_tool_call_params(choice.message)
),
)
# Mask tool call arguments
if hasattr(choice.message, "tool_calls") and choice.message.tool_calls:
for tool_call in choice.message.tool_calls:

5
litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py
View file

@ -26,6 +26,7 @@ from litellm.integrations.custom_guardrail import (
log_guardrail_information,
)
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.llms.anthropic.common_utils import message_dict_with_server_tool_calls
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
@ -430,7 +431,9 @@ class PillarGuardrail(CustomGuardrail):
# Extract response messages in the format Pillar expects
response_dict = response.model_dump() if hasattr(response, "model_dump") else {}
response_messages: Final = [
choice.get("message") for choice in response_dict.get("choices", []) if choice.get("message")
message_dict_with_server_tool_calls(choice["message"])
for choice in response_dict.get("choices", [])
if choice.get("message")
]
if not response_messages:

88
litellm/proxy/guardrails/guardrail_hooks/presidio.py
View file

@ -12,7 +12,7 @@ import asyncio
import json
import re
import threading
from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator, Awaitable, Iterator, Sequence
from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator, Awaitable, Callable, Iterator, Sequence
from contextlib import asynccontextmanager
from dataclasses import dataclass
from datetime import datetime
@ -43,6 +43,11 @@ from litellm.integrations.custom_guardrail import (
CustomGuardrail,
log_guardrail_information,
)
from litellm.llms.anthropic.common_utils import (
anthropic_server_tool_call_params,
with_server_tool_call_arguments,
write_anthropic_server_tool_calls,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames
from litellm.proxy.guardrails.anthropic_sse import (
@ -1392,6 +1397,18 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
verbose_proxy_logger.debug("No pii_tokens found in request_data['metadata'] — nothing to unmask")
presidio_config: Final = self.get_presidio_settings_from_request_data(request_data or {})
async def pii_processed_text(text: str) -> str:
match mode:
case "unmask":
return self._unmask_pii_text(text, pii_tokens)
case "mask":
return await self.check_pii(
text=text,
output_parse_pii=False,
presidio_config=presidio_config,
request_data=request_data,
)
for choice in response.choices:
message: Message | None = getattr(choice, "message", None)
if message is None:
@ -1400,15 +1417,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
# 1. Process content
content = getattr(message, "content", None)
if isinstance(content, str):
if mode == "unmask":
message.content = self._unmask_pii_text(content, pii_tokens)
elif mode == "mask":
message.content = await self.check_pii(
text=content,
output_parse_pii=False,
presidio_config=presidio_config,
request_data=request_data,
)
message.content = await pii_processed_text(content)
elif isinstance(content, list):
for item in content:
if not isinstance(item, dict):
@ -1416,54 +1425,43 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
text_value = item.get("text")
if text_value is None:
continue
if mode == "unmask":
item["text"] = self._unmask_pii_text(text_value, pii_tokens)
elif mode == "mask":
item["text"] = await self.check_pii(
text=text_value,
output_parse_pii=False,
presidio_config=presidio_config,
request_data=request_data,
)
item["text"] = await pii_processed_text(text_value)
await self._process_server_tool_calls_for_pii(message, pii_processed_text)
# 2. Process tool calls
for tool_call in attribute_items(message, "tool_calls"):
function = attribute_or_none(tool_call, "function")
args = attribute_or_none(function, "arguments") if function else None
if isinstance(args, str):
if mode == "unmask":
_write_attribute(function, "arguments", self._unmask_pii_text(args, pii_tokens))
elif mode == "mask":
_write_attribute(
function,
"arguments",
await self.check_pii(
text=args,
output_parse_pii=False,
presidio_config=presidio_config,
request_data=request_data,
),
)
_write_attribute(function, "arguments", await pii_processed_text(args))
# 3. Process legacy function calls
function_call = attribute_or_none(message, "function_call")
legacy_args = attribute_or_none(function_call, "arguments") if function_call else None
if isinstance(legacy_args, str):
if mode == "unmask":
_write_attribute(function_call, "arguments", self._unmask_pii_text(legacy_args, pii_tokens))
elif mode == "mask":
_write_attribute(
function_call,
"arguments",
await self.check_pii(
text=legacy_args,
output_parse_pii=False,
presidio_config=presidio_config,
request_data=request_data,
),
)
_write_attribute(function_call, "arguments", await pii_processed_text(legacy_args))
return response
@staticmethod
async def _process_server_tool_calls_for_pii(
message: Message,
pii_processed_text: Callable[[str], Awaitable[str]],
) -> None:
server_tool_calls: Final = anthropic_server_tool_call_params(message)
if not server_tool_calls:
return
processed_arguments: Final = tuple(
[
await pii_processed_text(server_tool_call["function"].get("arguments", ""))
for server_tool_call in server_tool_calls
]
)
write_anthropic_server_tool_calls(
message,
tuple(map(with_server_tool_call_arguments, server_tool_calls, processed_arguments)),
)
async def _mask_output_response(
self,
response: ModelResponse | EmbeddingResponse | ImageResponse,

13
litellm/proxy/guardrails/guardrail_hooks/repelloai/repelloai.py
View file

@ -12,6 +12,7 @@ from pydantic import BaseModel, TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.llms.anthropic.common_utils import tool_calls_with_server_tool_calls
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType]
httpxSpecialProvider,
@ -80,13 +81,11 @@ class RepelloAIGuardrail(CustomGuardrail):
def _extract_tool_call_args_from_message(cls, message: object) -> list[str]:
args: Final[list[str]] = []
tool_calls: Final = cls._get_field(message, "tool_calls")
if _is_object_list(tool_calls):
for tool_call in tool_calls:
function = cls._get_field(tool_call, "function")
arguments = cls._get_field(function, "arguments")
if isinstance(arguments, str) and arguments.strip():
args.append(arguments)
for tool_call in tool_calls_with_server_tool_calls(message):
function = cls._get_field(tool_call, "function")
arguments = cls._get_field(function, "arguments")
if isinstance(arguments, str) and arguments.strip():
args.append(arguments)
function_call: Final = cls._get_field(message, "function_call")
arguments = cls._get_field(function_call, "arguments")

20
litellm/proxy/guardrails/guardrail_hooks/tool_permission.py
View file

@ -19,6 +19,7 @@ from litellm.integrations.custom_guardrail import (
CustomGuardrail,
log_guardrail_information,
)
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params, anthropic_server_tool_calls
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.callback_utils import (
add_guardrail_to_applied_guardrails_header,
@ -516,7 +517,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
for choice_index, choice in enumerate(response.choices):
if isinstance(choice, Choices):
for tool in choice.message.tool_calls or []:
for tool in (*(choice.message.tool_calls or ()), *anthropic_server_tool_calls(choice.message)):
tool_calls.append(tool)
legacy_tool_call = self._legacy_function_call_to_tool_call(
getattr(choice.message, "function_call", None), choice_index
@ -791,6 +792,23 @@ class ToolPermissionGuardrail(CustomGuardrail):
choice.message.tool_calls = filtered_tool_calls if filtered_tool_calls else None
server_error_results = tuple(
error_results.get(self._judged_tool_call(server_tool_call))
for server_tool_call in anthropic_server_tool_calls(choice.message)
)
if any(server_error_results):
error_messages.extend(result.content for result in server_error_results if result is not None)
choice.message.provider_specific_fields = {
**(choice.message.provider_specific_fields or {}),
"server_tool_calls": [
server_tool_call
for server_tool_call, result in zip(
anthropic_server_tool_call_params(choice.message), server_error_results
)
if result is None
],
}
legacy_tool_call = self._legacy_function_call_to_tool_call(
getattr(choice.message, "function_call", None), choice_index
)

18
litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py
View file

@ -25,6 +25,7 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route
from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket
from litellm.llms import get_guardrail_translation_mapping, load_guardrail_translation_mappings
from litellm.llms.anthropic.common_utils import anthropic_server_call_fields
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.utils import (
@ -144,6 +145,15 @@ def _ends_stream(endpoint_translation: object, item: object) -> bool:
return has_ended is not None and has_ended((item,))
def _passthrough_delta(delta: object) -> Delta:
server_call_fields: Final = anthropic_server_call_fields(delta)
role: Final = getattr(delta, "role", None)
tool_calls: Final = getattr(delta, "tool_calls", None)
if not server_call_fields:
return Delta(content=None, role=role, tool_calls=tool_calls)
return Delta(content=None, role=role, tool_calls=tool_calls, provider_specific_fields=dict(server_call_fields))
def _held_choices(held_chars_per_choice: Mapping[int, int]) -> frozenset[int]:
return frozenset(idx for idx, held in held_chars_per_choice.items() if held > 0)
@ -974,7 +984,7 @@ class UnifiedLLMGuardrails(CustomLogger):
def _chunk_has_tool_calls(item: object) -> bool:
for choice in _chunk_choices(item):
delta = getattr(choice, "delta", None)
if getattr(delta, "tool_calls", None):
if getattr(delta, "tool_calls", None) or anthropic_server_call_fields(delta):
return True
return False
@ -1042,11 +1052,7 @@ class UnifiedLLMGuardrails(CustomLogger):
synthetic_choices.append(
StreamingChoices(
index=idx,
delta=Delta(
content=None,
role=getattr(delta, "role", None),
tool_calls=getattr(delta, "tool_calls", None),
),
delta=_passthrough_delta(delta),
finish_reason=passthrough_finish,
)
)

21
litellm/responses/litellm_completion_transformation/streaming_iterator.py
View file

@ -9,6 +9,7 @@ import litellm
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items
from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend
from litellm.llms.anthropic.common_utils import anthropic_server_tool_calls
from litellm.main import stream_chunk_builder
from litellm.responses.litellm_completion_transformation.compaction_items import compaction_output_item
from litellm.responses.litellm_completion_transformation.custom_tools import (
@ -486,13 +487,16 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
try:
message: Final = litellm_complete_object.choices[0].message
tool_calls = getattr(message, "tool_calls", None)
server_tool_calls: Final = anthropic_server_tool_calls(message)
tool_calls: Final = [*server_tool_calls, *(getattr(message, "tool_calls", None) or ())]
except Exception:
tool_calls = None
if not tool_calls or not isinstance(tool_calls, list):
return
if not tool_calls:
return
server_tool_call_ids: Final = frozenset(tool_call.id for tool_call in server_tool_calls)
for tc in tool_calls:
call_id_raw = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)
if not call_id_raw:
@ -518,6 +522,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
if (search_execution := self._tool_search_execution(call_id, fn_name)) is not None:
self._queue_tool_search_call_done(call_id, output_index, fn_args, search_execution)
continue
if call_id in server_tool_call_ids:
continue
# Track if this is a new tool call that wasn't streamed
is_new_tool_call = call_id not in self._tool_item_id_by_call_id
@ -592,10 +598,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
}
),
)
finished_events: Final = (
()
if item.status == "in_progress"
else ((ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),)
)
for event_type, event_class in (
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, WebSearchCallInProgressEvent),
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, WebSearchCallSearchingEvent),
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),
*finished_events,
):
self._sequence_number += 1
event = event_class(type=event_type, output_index=output_index, item_id=item.id)

77
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -56,7 +56,12 @@ from litellm.litellm_core_utils.hidden_params import (
set_hidden_params,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control
from litellm.llms.anthropic.common_utils import (
anthropic_server_tool_call_params,
anthropic_server_tool_calls,
supports_anthropic_cache_control,
validated_object_mapping,
)
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support
from litellm.llms.openrouter.chat.transformation import openrouter_model_supports_cache_control
@ -66,11 +71,12 @@ from litellm.responses.litellm_completion_transformation.session_handler import
ResponsesSessionHandler,
StoredTurns,
)
from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex
from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex, claude_hosted_web_tool_call
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
ChatCompletionAssistantToolCall,
ChatCompletionAudioObject,
ChatCompletionCachedContent,
ChatCompletionFileObject,
@ -875,6 +881,22 @@ def _chain_session_metadata(chain_session: str, request_kwargs: Mapping[str, obj
return {key: {**_STR_KEY_DICT_ADAPTER.validate_python(metadata), CALLER_SESSION_ID_METADATA_KEY: chain_session}}
def claude_web_search_call_tool_call(search: ResponseFunctionWebSearch) -> ChatCompletionAssistantToolCall | None:
claude_call: Final = claude_hosted_web_tool_call(search)
if claude_call is None:
return None
tool_id, name, arguments = claude_call
return ChatCompletionAssistantToolCall(
id=tool_id,
type="function",
function=ChatCompletionToolCallFunctionChunk(name=name, arguments=arguments),
)
def _pending_claude_server_tool_call(search: ResponseFunctionWebSearch) -> ChatCompletionAssistantToolCall | None:
return claude_web_search_call_tool_call(search) if search.status == "in_progress" else None
class LiteLLMCompletionResponsesConfig:
@staticmethod
def get_supported_openai_params(model: str) -> list:
@ -1780,10 +1802,22 @@ class LiteLLMCompletionResponsesConfig:
message_cache_control: Final = (
None if isinstance(content, list) else new_cache_control or previous_cache_control
)
new_server_tool_calls: Final = anthropic_server_tool_call_params(new_message)
server_tool_call_fields: Final = (
{
"provider_specific_fields": {
**(validated_object_mapping(last_message.get("provider_specific_fields")) or {}),
"server_tool_calls": [*anthropic_server_tool_call_params(last_message), *new_server_tool_calls],
}
}
if new_server_tool_calls
else {}
)
merged: Final = {
**{key: value for key, value in last_message.items() if key != "cache_control"},
"content": content,
**({} if message_cache_control is None else {"cache_control": message_cache_control}),
**server_tool_call_fields,
}
return cast( # cast-ok: preserves the assistant fields and content blocks
ChatCompletionAssistantMessage, merged
@ -2225,10 +2259,15 @@ class LiteLLMCompletionResponsesConfig:
content="Hosted web search: " + replay.model_dump_json(exclude_none=True),
)
]
hosted_search_text: Final = "Hosted web search: " + search.model_dump_json(exclude_none=True)
pending_server_tool_call: Final = _pending_claude_server_tool_call(search)
if pending_server_tool_call is None:
return [GenericChatCompletionMessage(role="assistant", content=hosted_search_text)]
return [
GenericChatCompletionMessage(
ChatCompletionResponseMessage(
role="assistant",
content="Hosted web search: " + search.model_dump_json(exclude_none=True),
content=hosted_search_text,
provider_specific_fields={"server_tool_calls": [pending_server_tool_call]},
)
]
if LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(input_item):
@ -2958,13 +2997,12 @@ class LiteLLMCompletionResponsesConfig:
all_chat_completion_tools: Final[list[ChatCompletionMessageToolCall]] = []
for choice in chat_completion_response.choices:
if isinstance(choice, Choices):
if choice.message.tool_calls:
all_chat_completion_tools.extend(choice.message.tool_calls)
for tool_call in choice.message.tool_calls:
TOOL_CALLS_CACHE.set_cache(
key=tool_call.id,
value=tool_call,
)
for tool_call in (*anthropic_server_tool_calls(choice.message), *(choice.message.tool_calls or ())):
all_chat_completion_tools.append(tool_call)
TOOL_CALLS_CACHE.set_cache(
key=tool_call.id,
value=tool_call,
)
request_tools: Final = responses_api_request.get("tools") if responses_api_request is not None else None
custom_tool_names: Final = extract_custom_tool_names(request_tools)
@ -3381,7 +3419,22 @@ class LiteLLMCompletionResponsesConfig:
for item in responses_output
]
return responses_output
server_tool_call_ids: Final = LiteLLMCompletionResponsesConfig._server_tool_call_ids(choices)
return [
item
for item in responses_output
if not (
getattr(item, "type", None) == "function_call"
and getattr(item, "call_id", None) in server_tool_call_ids
)
]
@staticmethod
def _server_tool_call_ids(choices: Sequence[Choices]) -> frozenset[str]:
server_tool_calls: Final = chain.from_iterable(
anthropic_server_tool_calls(choice.message) for choice in choices
)
return frozenset(tool_call.id for tool_call in server_tool_calls)
@staticmethod
def _extract_tool_result_output_items(

7
litellm/tool_loop.py
View file

@ -120,9 +120,12 @@ def _replays_provider_specific_fields(
client_tool_calls: tuple[ChatCompletionMessageToolCall, ...],
) -> bool:
from litellm.completion_extras.litellm_responses_transformation.transformation import carries_tool_search_items
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
return client_tool_calls != tool_calls or carries_tool_search_items(
getattr(message, "provider_specific_fields", None)
return (
client_tool_calls != tool_calls
or bool(anthropic_server_tool_call_params(message))
or carries_tool_search_items(getattr(message, "provider_specific_fields", None))
)

11
litellm/types/llms/anthropic.py
View file

@ -2,10 +2,12 @@ from collections.abc import Iterable, Mapping, Sequence
from enum import Enum
from typing import Final, Literal, TypeAlias
from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch
from pydantic import ConfigDict
from typing_extensions import NotRequired, ReadOnly, Required, TypedDict
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.responses.main import hosted_web_tool_call, web_search_call_tool_id
from .openai import (
ChatCompletionCachedContent,
@ -858,6 +860,15 @@ ANTHROPIC_TOOL_SEARCH_TOOL_TYPES: Final = frozenset(
ANTHROPIC_TOOL_SEARCH_TOOL_NAMES: Final = frozenset({"tool_search_tool_regex", "tool_search_tool_bm25"})
ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX: Final = "srvtoolu_"
def claude_hosted_web_tool_call(search: ResponseFunctionWebSearch) -> tuple[str, str, str] | None:
tool_id: Final = web_search_call_tool_id(search.id)
if tool_id is None or not tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX):
return None
hosted_call: Final = hosted_web_tool_call(search.action)
return None if hosted_call is None else (tool_id, *hosted_call)
# Effort beta header constant
ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24"

120
litellm/types/responses/main.py
View file

@ -1,10 +1,17 @@
import builtins
import json
from collections.abc import Mapping, Sequence
from typing import Final, Literal, Optional, Union
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
from openai.types.responses.response_function_web_search import ActionSearchSource, ResponseFunctionWebSearch
from pydantic import ConfigDict, PrivateAttr
from openai.types.responses.response_function_web_search import (
Action,
ActionOpenPage,
ActionSearch,
ActionSearchSource,
ResponseFunctionWebSearch,
)
from pydantic import ConfigDict, PrivateAttr, TypeAdapter, ValidationError
from typing_extensions import TypedDict
from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject
@ -42,14 +49,11 @@ class OutputFunctionToolCall(BaseLiteLLMOpenAIResponseObject):
phase: Phase = None
def build_web_search_call(
tool_id: str,
tool_input: object,
result: object,
status: Literal["in_progress", "searching", "completed", "failed"] | None = None,
) -> ResponseFunctionWebSearch:
HOSTED_WEB_TOOL_NAMES: Final = frozenset({"web_search", "web_fetch"})
def _web_search_action(tool_input: object, content: object) -> ActionSearch:
query: Final = tool_input.get("query", "") if isinstance(tool_input, Mapping) else ""
content: Final = result.get("content") if isinstance(result, Mapping) else None
result_items: Final = content if isinstance(content, Sequence) and not isinstance(content, (str, bytes)) else ()
sources: Final = [
ActionSearchSource(type="url", url=url)
@ -58,20 +62,96 @@ def build_web_search_call(
and item.get("type") == "web_search_result"
and isinstance((url := item.get("url")), str)
]
failed: Final = isinstance(content, Mapping) and content.get("type") == "web_search_tool_result_error"
return ResponseFunctionWebSearch(
id=f"ws_{tool_id}",
type="web_search_call",
status=status or ("failed" if failed else "completed"),
action={
"type": "search",
"query": query if isinstance(query, str) else "",
"queries": [query] if isinstance(query, str) and query else [],
"sources": sources,
},
return ActionSearch(
type="search",
query=query if isinstance(query, str) else "",
queries=[query] if isinstance(query, str) and query else [],
sources=sources,
)
def _web_fetch_action(tool_input: object) -> ActionOpenPage:
match tool_input:
case {"url": str() as url}:
return ActionOpenPage(type="open_page", url=url)
case _:
return ActionOpenPage(type="open_page", url=None)
def _result_content(result: object) -> object:
match result:
case {"content": object() as content}:
return content
case _:
return None
def _is_tool_result_error(content: object, tool_name: str) -> bool:
match content:
case {"type": str() as content_type}:
return content_type == f"{tool_name}_tool_result_error"
case _:
return False
WEB_SEARCH_CALL_ID_PREFIX: Final = "ws_"
def build_web_search_call(
tool_id: str,
tool_input: object,
result: object,
status: Literal["in_progress", "searching", "completed", "failed"] | None = None,
tool_name: str = "web_search",
) -> ResponseFunctionWebSearch:
content: Final = _result_content(result)
return ResponseFunctionWebSearch(
id=f"{WEB_SEARCH_CALL_ID_PREFIX}{tool_id}",
type="web_search_call",
status=status or ("failed" if _is_tool_result_error(content, tool_name) else "completed"),
action=_web_fetch_action(tool_input) if tool_name == "web_fetch" else _web_search_action(tool_input, content),
)
def web_search_call_tool_id(call_id: str) -> str | None:
tool_id: Final = call_id.removeprefix(WEB_SEARCH_CALL_ID_PREFIX)
return None if tool_id == call_id else tool_id
_HOSTED_WEB_TOOL_ARGUMENTS: Final = TypeAdapter(dict[str, object])
def hosted_web_tool_call(action: object) -> tuple[str, str] | None:
match action:
case ActionSearch(query=str() as query):
return "web_search", json.dumps({"query": query})
case ActionOpenPage(url=str() as url):
return "web_fetch", json.dumps({"url": url})
case _:
return None
def _decoded_hosted_web_tool_arguments(arguments: str) -> Mapping[str, object]:
try:
return _HOSTED_WEB_TOOL_ARGUMENTS.validate_json(arguments)
except ValidationError:
return {}
def with_hosted_web_tool_arguments(action: Action, arguments: str) -> Action:
match action, _decoded_hosted_web_tool_arguments(arguments):
case ActionSearch(), {"query": str() as query}:
return action.model_copy(update={"query": query, "queries": [query] if query else []})
case ActionSearch(), _:
return action.model_copy(update={"query": "", "queries": []})
case ActionOpenPage(), {"url": str() as url}:
return action.model_copy(update={"url": url})
case ActionOpenPage(), _:
return action.model_copy(update={"url": None})
case _:
return action
class OutputImageGenerationCall(BaseLiteLLMOpenAIResponseObject):
"""An image generation call output"""

67
tests/unit/integrations/arize/test_arize_utils.py
View file

@ -725,6 +725,73 @@ def test_arize_output_value_falls_back_to_tool_calls_summary():
assert "SF" in out
def test_arize_server_tool_calls_lead_the_output_summary_and_tool_call_attributes() -> None:
span: Final = TracerProvider().get_tracer(__name__).start_span("litellm_request")
kwargs: Final = {
"model": "claude-opus-5-5",
"messages": [{"role": "user", "content": "persimmon facts?"}],
"standard_logging_object": {"model_parameters": {}, "metadata": {}, "call_type": "completion"},
"optional_params": {},
"litellm_params": {"custom_llm_provider": "anthropic"},
}
response_obj: Final = ModelResponse(
usage={"total_tokens": 10, "completion_tokens": 4, "prompt_tokens": 6},
choices=[
Choices(
index=0,
finish_reason="tool_calls",
message=litellm.Message(
role="assistant",
content=None,
tool_calls=[
{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city": "SF"}'},
}
],
provider_specific_fields={
"server_tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "persimmon"}'},
}
]
},
),
)
],
model="claude-opus-5-5",
id="r-server-tools",
)
ArizeLogger.set_arize_attributes(span, kwargs, response_obj)
tool_calls_prefix: Final = f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_TOOL_CALLS}."
tool_call_attributes: Final = {
key: value for key, value in span.attributes.items() if key.startswith(tool_calls_prefix)
}
assert json.loads(span.attributes[SpanAttributes.OUTPUT_VALUE]) == {
"tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "persimmon"}'},
},
{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": '{"city": "SF"}'}},
]
}
assert tool_call_attributes == {
f"{tool_calls_prefix}0.{ToolCallAttributes.TOOL_CALL_ID}": "srvtoolu_01",
f"{tool_calls_prefix}0.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}": "web_search",
f"{tool_calls_prefix}0.{ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON}": '{"query": "persimmon"}',
f"{tool_calls_prefix}1.{ToolCallAttributes.TOOL_CALL_ID}": "call_1",
f"{tool_calls_prefix}1.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}": "get_weather",
f"{tool_calls_prefix}1.{ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON}": '{"city": "SF"}',
}
def test_arize_output_value_unchanged_when_content_present():
"""Regression guard: when content is non-empty, OUTPUT_VALUE must be
exactly that content (no summary written)."""

35
tests/unit/integrations/datadog/test_datadog_llm_obs.py
View file

@ -110,6 +110,41 @@ def test_output_tool_calls_use_the_datadog_tool_call_schema(logger: DataDogLLMOb
assert "function" not in message["tool_calls"][0]
def test_claude_server_tool_calls_are_reported_before_client_tool_calls(logger: DataDogLLMObsLogger) -> None:
payload = build(
logger,
response_message={
"role": "assistant",
"content": None,
"tool_calls": [ASSISTANT_TOOL_CALL],
"provider_specific_fields": {
"server_tool_calls": [
{
"id": "srvtoolu_01Search",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "paris weather"}'},
}
]
},
},
)
assert payload["meta"]["output"]["messages"][0]["tool_calls"] == [
{
"name": "web_search",
"arguments": {"query": "paris weather"},
"tool_id": "srvtoolu_01Search",
"type": "function",
},
{
"name": "get_weather",
"arguments": {"city": "Paris", "unit": "c"},
"tool_id": "call_abc123",
"type": "function",
},
]
def test_tool_call_identifiers_that_are_not_strings_are_blanked_not_stringified(logger: DataDogLLMObsLogger) -> None:
payload = build(
logger,

20
tests/unit/integrations/newrelic/test_newrelic.py
View file

@ -1,3 +1,4 @@
import json
import os
import sys
from datetime import datetime, timezone
@ -45,6 +46,12 @@ NR_ENV = {
"NEW_RELIC_APP_NAME": "test-app",
}
_SERVER_CALL: Final = {
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "persimmon"}'},
}
def make_logger(**kwargs) -> NewRelicLogger:
"""Instantiate NewRelicLogger with NR agent calls mocked out."""
@ -361,6 +368,19 @@ class TestExtractMessageContent:
assert "get_weather" in result
assert "call_1" in result
@pytest.mark.parametrize(
"message",
[
{"role": "assistant", "content": None, "provider_specific_fields": {"server_tool_calls": [_SERVER_CALL]}},
litellm.Message(
role="assistant", content=None, provider_specific_fields={"server_tool_calls": [_SERVER_CALL]}
),
],
ids=["dict", "message"],
)
def test_server_tool_call_only_message_serializes_the_server_call(self, message):
assert json.loads(self.logger._extract_message_content(message)) == [_SERVER_CALL]
def test_multimodal_list_serialized_as_json(self):
msg = {
"content": [

32
tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py
View file

@ -48,6 +48,7 @@ from litellm.integrations.otel.model.spans import (
validate_registry,
)
from litellm.integrations.otel.model.trace_controls import TraceControls, caller_trace_controls
from litellm.types.responses.main import build_web_search_call
@pytest.fixture(autouse=True)
@ -798,6 +799,37 @@ def test_responses_tool_calls_fold_into_the_assistant_message_with_tool_calls_fi
assert data.finish_reasons == ("tool_calls",)
def test_responses_claude_web_calls_lead_the_tool_calls_without_ending_the_turn_on_tool_calls():
claude_fetch = build_web_search_call("srvtoolu_01", {"url": "https://example.com/"}, None, tool_name="web_fetch")
openai_search = {
"type": "web_search_call",
"id": "ws_1",
"status": "completed",
"action": {"type": "search", "query": "news"},
}
client_call = {"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": "{}"}
claude_tool_call = {
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_fetch", "arguments": '{"url": "https://example.com/"}'},
}
hosted_only = LLMCallSpanData.from_standard_logging_payload(
_responses_payload([_RESPONSES_TEXT_ITEM, claude_fetch.model_dump(), openai_search]), capture_content=True
)
with_client_call = LLMCallSpanData.from_standard_logging_payload(
_responses_payload([_RESPONSES_TEXT_ITEM, claude_fetch.model_dump(), client_call]), capture_content=True
)
assert json.loads(json.dumps(hosted_only.choices_out[0]["message"]["tool_calls"])) == [claude_tool_call]
assert hosted_only.finish_reasons == ("stop",)
assert json.loads(json.dumps(with_client_call.choices_out[0]["message"]["tool_calls"])) == [
claude_tool_call,
{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}},
]
assert with_client_call.finish_reasons == ("tool_calls",)
def test_responses_tool_call_only_output_has_no_content():
data = LLMCallSpanData.from_standard_logging_payload(
_responses_payload([{"type": "function_call", "id": "fc_1", "name": "get_weather", "arguments": "{}"}]),

39
tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py
View file

@ -174,6 +174,45 @@ def test_openinference_output_tool_calls_preserve_calls_in_attributes_and_value(
]
def test_openinference_output_lists_claude_server_tool_calls_before_client_tool_calls():
data: Final = _llm_call(
choices_out=(
{
"finish_reason": "tool_calls",
"message": {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_paris",
"type": "function",
"function": {"name": "lookup_weather", "arguments": '{"city": "Paris"}'},
}
],
"provider_specific_fields": {
"server_tool_calls": [
{
"id": "srvtoolu_01Search",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "paris weather"}'},
}
]
},
},
},
)
)
attrs: Final = OpenInferenceMapper().map(data)
assert {key: value for key, value in attrs.items() if ".tool_calls." in key} == {
"llm.output_messages.0.message.tool_calls.0.tool_call.id": "srvtoolu_01Search",
"llm.output_messages.0.message.tool_calls.0.tool_call.function.name": "web_search",
"llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": '{"query": "paris weather"}',
"llm.output_messages.0.message.tool_calls.1.tool_call.id": "call_paris",
"llm.output_messages.0.message.tool_calls.1.tool_call.function.name": "lookup_weather",
"llm.output_messages.0.message.tool_calls.1.tool_call.function.arguments": '{"city": "Paris"}',
}
def test_openinference_responses_tool_calls_are_emitted_as_output_attributes():
data: Final = LLMCallSpanData.from_standard_logging_payload(
_responses_payload(

45
tests/unit/integrations/test_helicone.py
View file

@ -17,7 +17,8 @@ def _claude_mapping(messages, response_obj):
return logger.claude_mapping(model="gpt-5.6", messages=messages, response_obj=response_obj)
def test_claude_mapping_serializes_custom_tool_calls(monkeypatch):
@pytest.fixture()
def anthropic_prompts(monkeypatch):
"""
Stub the anthropic module unconditionally: the SDK may be absent (it lives in the
proxy-runtime extra), and the tests/unit/llms/anthropic test package can
@ -27,6 +28,10 @@ def test_claude_mapping_serializes_custom_tool_calls(monkeypatch):
stub.HUMAN_PROMPT = "\n\nHuman:"
stub.AI_PROMPT = "\n\nAssistant:"
monkeypatch.setitem(sys.modules, "anthropic", stub)
@pytest.mark.usefixtures("anthropic_prompts")
def test_claude_mapping_serializes_custom_tool_calls():
response_obj = {
"id": "chatcmpl-1",
"choices": [
@ -58,6 +63,44 @@ def test_claude_mapping_serializes_custom_tool_calls(monkeypatch):
assert {"type": "tool_use", "id": "call_f", "name": "read_file", "input": '{"path": "a.py"}'} in tool_use_blocks
@pytest.mark.usefixtures("anthropic_prompts")
def test_claude_mapping_puts_server_tool_calls_first_as_tool_use_blocks():
response_obj = {
"id": "chatcmpl-1",
"choices": [
{
"message": {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_f",
"type": "function",
"function": {"name": "read_file", "arguments": '{"path": "a.py"}'},
},
],
"provider_specific_fields": {
"server_tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "persimmon"}'},
}
]
},
},
"finish_reason": "tool_calls",
}
],
"usage": {"prompt_tokens": 1, "completion_tokens": 2},
}
mapped = _claude_mapping([{"role": "user", "content": "hi"}], response_obj)
assert mapped["content"] == [
{"type": "tool_use", "id": "srvtoolu_01", "name": "web_search", "input": '{"query": "persimmon"}'},
{"type": "tool_use", "id": "call_f", "name": "read_file", "input": '{"path": "a.py"}'},
]
@pytest.fixture()
def _vcr_outcome_gate(request, vcr):
install_live_call_probe(request, vcr)

49
tests/unit/integrations/test_langfuse_otel.py
View file

@ -1309,6 +1309,55 @@ class TestDerivedTraceFields:
assert json.loads(str(attributes["langfuse.observation.output"])) == {"content": "hi"}
def test_server_tool_calls_lead_the_observation_output(self) -> None:
response_obj: Final = ModelResponse(
id="chatcmpl-server",
choices=[
Choices(
index=0,
finish_reason="tool_calls",
message=litellm.Message(
role="assistant",
content=None,
tool_calls=[
{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city": "SF"}'},
}
],
provider_specific_fields={
"server_tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "persimmon"}'},
}
]
},
),
)
],
)
attributes: Final = _emitted({"call_type": "acompletion", "litellm_params": {"metadata": {}}}, response_obj)
assert json.loads(str(attributes["langfuse.observation.output"])) == [
{
"id": "chatcmpl-server",
"name": "web_search",
"call_id": "srvtoolu_01",
"type": "function_call",
"arguments": {"query": "persimmon"},
},
{
"id": "chatcmpl-server",
"name": "get_weather",
"call_id": "call_1",
"type": "function_call",
"arguments": {"city": "SF"},
},
]
def test_no_messages_leaves_trace_input_unset_but_emits_output(self) -> None:
attributes: Final = _emitted(
{"call_type": "acompletion", "litellm_params": {"metadata": {}}}, _chat_response("hi")

46
tests/unit/integrations/test_lunary.py
View file

@ -1,6 +1,7 @@
from litellm.integrations.lunary import parse_tool_calls
import litellm
from litellm.integrations.lunary import parse_messages, parse_tool_calls
from litellm.types.utils import (
ChatCompletionMessageCustomToolCall,
ChatCompletionMessageToolCall,
@ -35,3 +36,46 @@ def test_parse_tool_calls_serializes_custom_tool_calls():
def test_parse_tool_calls_none_passthrough():
assert parse_tool_calls(None) is None
def test_parse_messages_lists_server_tool_calls_before_client_calls():
response = litellm.ModelResponse(
choices=[
litellm.Choices(
index=0,
finish_reason="tool_calls",
message=litellm.Message(
role="assistant",
content=None,
tool_calls=[
{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city": "SF"}'},
}
],
provider_specific_fields={
"server_tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "persimmon"}'},
}
]
},
),
)
],
)
assert parse_messages(response.choices) == {
"role": "assistant",
"content": None,
"tool_calls": [
{
"type": "function",
"id": "srvtoolu_01",
"function": {"name": "web_search", "arguments": '{"query": "persimmon"}'},
},
{"type": "function", "id": "call_1", "function": {"name": "get_weather", "arguments": '{"city": "SF"}'}},
],
}

54
tests/unit/integrations/test_opentelemetry.py
View file

@ -372,6 +372,60 @@ def test_v1_routing_diagnostics_add_scalar_attributes_and_preserve_legacy_metada
assert finished.attributes["metadata.routing_decision"] == str(decision)
def test_v1_server_tool_calls_lead_the_completion_function_call_attributes() -> None:
exporter: Final = InMemorySpanExporter()
provider: Final = TracerProvider()
provider.add_span_processor(SimpleSpanProcessor(exporter))
otel: Final = OpenTelemetry(tracer_provider=provider)
response_obj: Final = litellm.ModelResponse(
choices=[
litellm.Choices(
index=0,
finish_reason="tool_calls",
message=litellm.Message(
role="assistant",
content=None,
tool_calls=[
{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city": "SF"}'},
}
],
provider_specific_fields={
"server_tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "persimmon"}'},
}
]
},
),
)
],
)
with provider.get_tracer(__name__).start_as_current_span("inference") as span:
otel.set_attributes(
span,
{
"model": "claude-opus-5-5",
"standard_logging_object": {"id": "server-tools-call", "call_type": "acompletion", "metadata": {}},
},
response_obj,
)
(finished,) = exporter.get_finished_spans()
function_call_attributes: Final = {
key: value for key, value in finished.attributes.items() if ".function_call." in key
}
assert function_call_attributes == {
f"{SpanAttributes.LLM_COMPLETIONS.value}.0.function_call.name": "web_search",
f"{SpanAttributes.LLM_COMPLETIONS.value}.0.function_call.arguments": '{"query": "persimmon"}',
f"{SpanAttributes.LLM_COMPLETIONS.value}.1.function_call.name": "get_weather",
f"{SpanAttributes.LLM_COMPLETIONS.value}.1.function_call.arguments": '{"city": "SF"}',
}
def test_v1_pod_routing_none_matches_other_none_metadata_attributes() -> None:
exporter: Final = InMemorySpanExporter()
provider: Final = TracerProvider()

34
tests/unit/integrations/test_shadow_eval_logger.py
View file

@ -1905,6 +1905,40 @@ class TestShadowPipeline:
assert "Let me look that up. [tool call] Read({})" in judge_prompt
async def test_a_server_tool_call_reaches_the_judge_ahead_of_client_tool_calls(self):
router = _shadow_reply_router(
{
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city": "SF"}'},
}
],
"provider_specific_fields": {
"server_tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "persimmon"}'},
}
]
},
},
finish_reason="tool_calls",
)
await self._judged_shadow_row(router)
judge_prompt = next(
call.kwargs["messages"][-1]["content"]
for call in router.acompletion.call_args_list
if call.kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != SHADOW_EVAL_ROUTER_CALL_ORIGIN
)
shadow_text: Final = '[tool call] web_search({"query": "persimmon"}), [tool call] get_weather({"city": "SF"})'
assert f":\n{shadow_text}\n\n" in judge_prompt, judge_prompt
async def test_an_empty_shadow_reply_names_the_finish_reason_and_the_routed_model(self):
"""A reply that really carried no text is diagnosable only if the row says what
the arm was doing when it produced none: a truncated turn and a model that answers

29
tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py
View file

@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
converse_redacted_content,
encrypted_content_from_signature,
encrypted_reasoning_signature,
get_content_from_model_response,
get_file_ids_from_messages,
get_format_from_file_id,
get_semantic_cache_prompt_from_messages,
@ -46,6 +47,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
with_prompt_cache_breakpoint,
)
from litellm.types.llms.openai import ChatCompletionAudioObject, ChatCompletionFileObject, ChatCompletionImageObject
from litellm.types.utils import ChatCompletionMessageToolCall, Choices, Function, Message, ModelResponse
_ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\]]{1,200}$'
_ARTIFACT_DATA_ID_PATTERN: Final = r"^(?!\.\.?(?:\/|$))[A-Za-z0-9_\-.~:@+]{1,200}$"
@ -2675,3 +2677,30 @@ def test_tpm_projection_preserves_non_media_identity_and_opaque_values() -> None
projected, media = split_media_for_tpm(request)
assert projected is request
assert media == ()
def test_get_content_from_model_response_includes_server_tool_calls_before_client_calls() -> None:
server_call: Final = ChatCompletionMessageToolCall(
id="srvtoolu_01", type="function", function=Function(name="web_search", arguments='{"query": "persimmon"}')
)
client_call: Final = ChatCompletionMessageToolCall(
id="call_1", type="function", function=Function(name="get_weather", arguments='{"city": "SF"}')
)
response: Final = ModelResponse(
choices=[
Choices(
index=0,
finish_reason="tool_calls",
message=Message(
role="assistant",
content="Looking it up.",
tool_calls=[client_call.model_dump()],
provider_specific_fields={"server_tool_calls": [server_call.model_dump()]},
),
)
],
)
assert get_content_from_model_response(response) == (
"Looking it up." + server_call.model_dump_json() + client_call.model_dump_json()
)

114
tests/unit/litellm_core_utils/test_redact_messages.py
View file

@ -20,10 +20,11 @@ from unittest.mock import patch
import httpx
import pytest
import pytest_asyncio
from pydantic import JsonValue
from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE
from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE, REDACTED_BY_LITELLM
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.litellm_core_utils.redact_messages import (
@ -37,6 +38,7 @@ from litellm.litellm_core_utils.redact_messages import (
)
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.responses.main import mock_responses_api_response
from litellm.types.responses.main import build_web_search_call
from litellm.types.router import BaselineRouteStamp
from litellm.types.utils import (
ModelResponse,
@ -169,6 +171,114 @@ def test_streaming_redaction_redacts_delta_provider_fields_without_touching_the_
assert fields == _provider_fields()
_SECRET_URL: Final = f"https://example.com/?q={SECRET}"
_WEB_SEARCH_CALL: Final = TypeAdapter(ResponseFunctionWebSearch)
def _claude_fetch_call() -> ResponseFunctionWebSearch:
return build_web_search_call(
"srvtoolu_01Fetch", {"url": _SECRET_URL}, {"content": []}, status="in_progress", tool_name="web_fetch"
)
def _web_call_summaries(calls: object) -> list[tuple[str, str | None, dict[str, object]]]:
items: Final = calls.values() if isinstance(calls, dict) else calls
return [
(call.id, call.status, call.action.model_dump(exclude_none=True))
for call in (_WEB_SEARCH_CALL.validate_python(item) for item in items)
]
@pytest.mark.parametrize("shape", ["object", "dict"])
def test_redaction_hides_claude_server_call_inputs_but_keeps_the_calls(shape: str) -> None:
response: Final = ModelResponse(
model="claude-opus-5-5",
choices=[
{
"message": {
"role": "assistant",
"content": None,
"provider_specific_fields": {
"server_tool_calls": [
{
"id": "srvtoolu_01Fetch",
"type": "function",
"function": {"name": "web_fetch", "arguments": json.dumps({"url": _SECRET_URL})},
}
],
"web_search_calls": [
_claude_fetch_call(),
build_web_search_call(
"srvtoolu_01Search", {"query": SECRET}, {"content": []}, status="in_progress"
),
],
},
}
}
],
)
result: Final = response if shape == "object" else response.model_dump()
redacted: Final = perform_redaction({}, result)
message: Final = (
redacted.choices[0].message if shape == "object" else SimpleNamespace(**redacted["choices"][0]["message"])
)
fields: Final = message.provider_specific_fields
assert fields["server_tool_calls"][0]["id"] == "srvtoolu_01Fetch"
assert fields["server_tool_calls"][0]["function"] == {"name": "web_fetch", "arguments": REDACTED_BY_LITELLM}
assert _web_call_summaries(fields["web_search_calls"]) == [
("ws_srvtoolu_01Fetch", "in_progress", {"type": "open_page", "url": REDACTED_BY_LITELLM}),
("ws_srvtoolu_01Search", "in_progress", {"type": "search", "query": REDACTED_BY_LITELLM}),
]
assert SECRET not in str(redacted), redacted
assert SECRET in str(result)
@pytest.mark.parametrize("key", ["complete_streaming_response", "async_complete_streaming_response"])
def test_streaming_redaction_hides_the_hosted_copy_of_a_claude_web_fetch(key: str) -> None:
response: Final = ModelResponseStream(
choices=[
litellm.utils.StreamingChoices(
delta=litellm.utils.Delta(
provider_specific_fields={"web_search_calls": {"srvtoolu_01Fetch": _claude_fetch_call()}}
)
)
]
)
perform_redaction({"stream": True, key: response}, None)
assert _web_call_summaries(response.choices[0].delta.provider_specific_fields["web_search_calls"]) == [
("ws_srvtoolu_01Fetch", "in_progress", {"type": "open_page", "url": REDACTED_BY_LITELLM})
]
@pytest.mark.parametrize("shape", ["object", "dict"])
def test_redaction_hides_web_search_call_actions_in_responses_api_output(shape: str) -> None:
response: Final = ResponsesAPIResponse(
id="resp_1",
created_at=1,
model="claude-opus-5-5",
object="response",
status="completed",
output=[
build_web_search_call("srvtoolu_01Fetch", {"url": _SECRET_URL}, {"content": []}, tool_name="web_fetch"),
build_web_search_call("srvtoolu_01Search", {"query": SECRET}, {"content": []}),
],
)
result: Final = response if shape == "object" else response.model_dump()
redacted: Final = perform_redaction({}, result)
output: Final = redacted.output if shape == "object" else redacted["output"]
assert _web_call_summaries(output) == [
("ws_srvtoolu_01Fetch", "completed", {"type": "open_page", "url": REDACTED_BY_LITELLM}),
("ws_srvtoolu_01Search", "completed", {"type": "search", "query": REDACTED_BY_LITELLM}),
]
assert SECRET in str(result)
@pytest.mark.parametrize("shape", ["object", "dict"])
def test_redaction_hides_a_compaction_summary_in_responses_api_output(shape: str) -> None:
response: Final = ResponsesAPIResponse(

118
tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py
View file

@ -15,7 +15,9 @@ import litellm
from litellm._uuid import uuid
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
from litellm.llms.anthropic.chat.handler import ModelResponseIterator, _envelope_verdict, make_call, make_sync_call
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
from litellm.llms.base_llm.base_utils import response_format_schema, unwrap_response_format_envelope
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.types.llms.openai import (
@ -23,7 +25,7 @@ from litellm.types.llms.openai import (
ChatCompletionToolCallFunctionChunk,
)
from litellm.types.responses.main import OutputCodeInterpreterCall
from litellm.types.utils import ChatCompletionDeltaToolCall, ModelResponseStream
from litellm.types.utils import ChatCompletionDeltaToolCall, ChatCompletionMessageToolCall, Choices, ModelResponseStream
from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE
@ -1780,19 +1782,113 @@ def test_web_search_tool_result_no_extra_tool_calls():
for tc in parsed.choices[0].delta.tool_calls:
tool_calls_emitted.append(tc)
# Should have exactly 2 tool calls:
# 1. From content_block_start (server_tool_use) with id and name
# 2. From content_block_delta with the actual query
assert len(tool_calls_emitted) == 2, f"Expected 2 tool calls, got {len(tool_calls_emitted)}"
assert tool_calls_emitted == []
# First tool call should have the id and name
assert tool_calls_emitted[0]["id"] == "srvtoolu_01ABC123"
assert tool_calls_emitted[0]["function"]["name"] == "web_search"
# Second tool call should have the query arguments
assert tool_calls_emitted[1]["function"]["arguments"] == '{"query": "test"}'
def _server_tool_input_delta(fragment: str) -> dict[str, object]:
return {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": fragment}}
# The {} chunks from web_search_tool_result should NOT have been emitted as tool calls
def _server_search_then_answer(*partial_json: str) -> tuple[dict[str, object], ...]:
return (
{
"type": "message_start",
"message": {"id": "msg_01", "model": "claude-opus-5-5", "usage": {"input_tokens": 9}},
},
{
"type": "content_block_start",
"index": 0,
"content_block": {"type": "server_tool_use", "id": "srvtoolu_01Web", "name": "web_search", "input": {}},
},
*(_server_tool_input_delta(fragment) for fragment in partial_json),
{"type": "content_block_stop", "index": 0},
{
"type": "content_block_start",
"index": 1,
"content_block": {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_01Web", "content": []},
},
{"type": "content_block_stop", "index": 1},
{"type": "content_block_start", "index": 2, "content_block": {"type": "text", "text": ""}},
{"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": "Paris is in France."}},
{"type": "content_block_stop", "index": 2},
)
_SERVER_SEARCH_THEN_ANSWER: Final = _server_search_then_answer('{"query": "Paris"}')
_CLIENT_TOOL_CALL: Final[tuple[dict[str, object], ...]] = (
{
"type": "content_block_start",
"index": 3,
"content_block": {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {}},
},
{
"type": "content_block_delta",
"index": 3,
"delta": {"type": "input_json_delta", "partial_json": '{"city": "Paris"}'},
},
{"type": "content_block_stop", "index": 3},
)
def _message_delta(stop_reason: str) -> dict[str, object]:
return {"type": "message_delta", "delta": {"stop_reason": stop_reason}, "usage": {"output_tokens": 20}}
_PARIS_SEARCH: Final = ("srvtoolu_01Web", "web_search", '{"query": "Paris"}')
@pytest.mark.parametrize(
("events", "expected_tool_calls", "expected_finish_reason", "expected_server_tool_call"),
[
pytest.param(
(*_SERVER_SEARCH_THEN_ANSWER, _message_delta("end_turn")), [], "stop", _PARIS_SEARCH, id="server_tool_only"
),
pytest.param(
(*_SERVER_SEARCH_THEN_ANSWER, *_CLIENT_TOOL_CALL, _message_delta("tool_use")),
[("toolu_01Weather", "get_weather", '{"city": "Paris"}')],
"tool_calls",
_PARIS_SEARCH,
id="server_tool_then_client_tool",
),
pytest.param(
(*_server_search_then_answer('{"que', 'ry": "Pa', 'ris"}'), _message_delta("end_turn")),
[],
"stop",
_PARIS_SEARCH,
id="server_tool_input_split_across_deltas",
),
pytest.param(
(*_server_search_then_answer(), _message_delta("end_turn")),
[],
"stop",
("srvtoolu_01Web", "web_search", "{}"),
id="server_tool_with_empty_input",
),
],
)
def test_streamed_server_tool_use_is_not_returned_as_a_client_tool_call(
events: tuple[dict[str, object], ...],
expected_tool_calls: list[tuple[str, str, str]],
expected_finish_reason: str,
expected_server_tool_call: tuple[str, str, str],
):
choice: Final = _assemble_stream(events).choices[0]
assert isinstance(choice, Choices)
assert (choice.message.tool_calls is None) == (not expected_tool_calls)
assert [
(call.id, call.function.name, call.function.arguments)
for call in choice.message.tool_calls or ()
if isinstance(call, ChatCompletionMessageToolCall)
] == expected_tool_calls
assert choice.finish_reason == expected_finish_reason
assert choice.message.content == "Paris is in France."
assert [
(call["id"], call["function"].get("name"), call["function"].get("arguments"))
for call in anthropic_server_tool_call_params(choice.message)
] == [expected_server_tool_call]
web_search_results: Final = message_field(choice.message.provider_specific_fields, "web_search_results")
assert [message_field(result, "tool_use_id") for result in parts_of(web_search_results)] == ["srvtoolu_01Web"]
def test_current_content_block_type_tracking():

222
tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py
View file

@ -1,8 +1,9 @@
import copy
import json
import logging
from collections.abc import Mapping
from itertools import chain
from typing import Final
from typing import Final, cast
from unittest.mock import MagicMock, patch
import httpx
@ -23,8 +24,13 @@ from litellm.constants import (
)
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
from litellm.llms.anthropic.chat import ModelResponseIterator
from litellm.llms.anthropic.common_utils import process_anthropic_headers
from litellm.llms.anthropic.common_utils import (
anthropic_server_tool_call_params,
process_anthropic_headers,
validated_object_mapping,
)
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema
from litellm.llms.anthropic.messages.native.transformation import (
@ -39,8 +45,8 @@ from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation im
VertexAIAnthropicConfig,
)
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES
from litellm.types.llms.openai import ChatCompletionToolCallFunctionChunk
from litellm.types.utils import ChatCompletionToolCallChunk, ModelResponse, ServerToolUse, Usage
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallFunctionChunk
from litellm.types.utils import ChatCompletionToolCallChunk, Choices, Message, ModelResponse, ServerToolUse, Usage
from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE
@ -3551,21 +3557,17 @@ def test_code_execution_tool_results_extraction():
prefix_prompt=None,
)
# Verify tool calls are present
assert transformed_response.choices[0].message.tool_calls is not None
assert len(transformed_response.choices[0].message.tool_calls) == 2
assert transformed_response.choices[0].message.tool_calls is None
# Verify first tool call
assert transformed_response.choices[0].message.tool_calls[0].id == "srvtoolu_01ABC"
assert transformed_response.choices[0].message.tool_calls[0].function.name == "bash_code_execution"
# Verify second tool call
assert transformed_response.choices[0].message.tool_calls[1].id == "srvtoolu_01DEF"
assert transformed_response.choices[0].message.tool_calls[1].function.name == "text_editor_code_execution"
# Verify tool results are in provider_specific_fields
provider_fields = transformed_response.choices[0].message.provider_specific_fields
assert provider_fields is not None
server_tool_calls: Final = anthropic_server_tool_call_params(transformed_response.choices[0].message)
assert [(call["id"], call["function"].get("name")) for call in server_tool_calls] == [
("srvtoolu_01ABC", "bash_code_execution"),
("srvtoolu_01DEF", "text_editor_code_execution"),
]
# Verify tool results are in provider_specific_fields
assert "tool_results" in provider_fields
assert provider_fields["tool_results"] is not None
assert len(provider_fields["tool_results"]) == 2
@ -3588,6 +3590,45 @@ def test_code_execution_tool_results_extraction():
assert "Done!" in transformed_response.choices[0].message.content
def test_server_and_client_tool_calls_split_by_block_type_not_id_prefix():
completion_response: Final = {
"id": "msg_split",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5-20250929",
"content": [
{"type": "server_tool_use", "id": "toolu_server", "name": "web_search", "input": {"query": "persimmon"}},
{"type": "tool_use", "id": "srvtoolu_client", "name": "get_weather", "input": {"city": "Paris"}},
],
"stop_reason": "tool_use",
"stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 5},
}
raw_response: Final = MagicMock(spec=httpx.Response)
raw_response.headers = {}
message: Final = (
AnthropicConfig()
.transform_parsed_response(
completion_response=completion_response,
raw_response=raw_response,
model_response=ModelResponse(),
json_mode=False,
prefix_prompt=None,
)
.choices[0]
.message
)
assert [
(call["id"], call["function"].get("name"), call["function"].get("arguments"))
for call in anthropic_server_tool_call_params(message)
] == [("toolu_server", "web_search", '{"query": "persimmon"}')]
assert [
(tool_call.id, tool_call.function.name, tool_call.function.arguments) for tool_call in message.tool_calls or ()
] == [("srvtoolu_client", "get_weather", '{"city": "Paris"}')]
def test_code_execution_tool_results_in_hidden_params():
"""
Test that tool_results reaches _hidden_params so the Responses API adapter
@ -3675,7 +3716,7 @@ def _tool_search_turn(tool_references: list[str], last_block: dict[str, object])
]
def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str):
def _parsed_choice(content: list[dict[str, object]], stop_reason: str) -> Choices:
anthropic_response: Final = {
"id": "msg_01XYZ",
"type": "message",
@ -3690,7 +3731,7 @@ def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str
raw_response.json.return_value = anthropic_response
raw_response.status_code = 200
raw_response.headers = {}
return (
choice: Final = (
AnthropicConfig()
.transform_parsed_response(
completion_response=anthropic_response,
@ -3700,14 +3741,19 @@ def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str
prefix_prompt=None,
)
.choices[0]
.message
)
assert isinstance(choice, Choices)
return choice
def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str) -> Message:
return _parsed_choice(content, stop_reason).message
_WEATHER_CALL: Final = {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}}
def _web_search_turn(last_block: dict[str, object]) -> list[dict[str, object]]:
def _web_search_turn(last_block: Mapping[str, object]) -> list[dict[str, object]]:
return [
{"type": "server_tool_use", "id": "srvtoolu_01Web", "name": "web_search", "input": {"query": "Paris facts"}},
{
@ -3723,7 +3769,7 @@ def _web_search_turn(last_block: dict[str, object]) -> list[dict[str, object]]:
}
],
},
last_block,
dict(last_block),
]
@ -3770,8 +3816,114 @@ def test_server_tool_result_round_trips_right_after_its_call(
assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"]
def _block_ids(message: dict[str, object], block_type: str, id_key: str) -> list[object]:
return sorted(block[id_key] for block in message["content"] if block["type"] == block_type)
_FETCH_STILL_WAITING_ON_ITS_RESULT: Final = {
"type": "server_tool_use",
"id": "srvtoolu_01Fetch",
"name": "web_fetch",
"input": {"url": "https://example.com/"},
}
@pytest.mark.parametrize("modify_params", [False, True])
def test_server_call_still_waiting_on_its_result_is_replayed_as_a_server_call(
modify_params: bool, monkeypatch: pytest.MonkeyPatch
):
content: Final = [_FETCH_STILL_WAITING_ON_ITS_RESULT, _WEATHER_CALL]
message: Final = _parsed_assistant_message(content, "tool_use")
monkeypatch.setattr(litellm, "modify_params", modify_params)
replayed: Final = _replayed(
{"role": "user", "content": "Fetch example.com, then tell me the weather in Paris"},
message.model_dump(),
{"role": "tool", "tool_call_id": "toolu_01Weather", "content": "18C"},
)
assert replayed[1]["content"] == content
assert [turn["role"] for turn in replayed] == ["user", "assistant", "user"]
assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"]
_PARIS_TEXT: Final = "Paris is the capital of France."
_PARIS_ANSWER: Final[Mapping[str, str]] = {"type": "text", "text": _PARIS_TEXT}
@pytest.mark.parametrize(
("content", "stop_reason", "expected_tool_call_ids", "expected_finish_reason"),
[
pytest.param(_web_search_turn(_PARIS_ANSWER), "end_turn", [], "stop", id="web_search_then_answer"),
pytest.param(
_web_search_turn(_WEATHER_CALL), "tool_use", ["toolu_01Weather"], "tool_calls", id="web_search_then_tool"
),
pytest.param([_WEATHER_CALL], "tool_use", ["toolu_01Weather"], "tool_calls", id="client_tool_only"),
],
)
def test_server_tool_use_is_not_returned_as_a_client_tool_call(
content: list[dict[str, object]],
stop_reason: str,
expected_tool_call_ids: list[str],
expected_finish_reason: str,
):
choice: Final = _parsed_choice(content, stop_reason)
assert (choice.message.tool_calls is None) == (not expected_tool_call_ids)
assert [call.id for call in choice.message.tool_calls or ()] == expected_tool_call_ids
assert choice.finish_reason == expected_finish_reason
assert [
(call["id"], call["function"].get("name"), call["function"].get("arguments"))
for call in anthropic_server_tool_call_params(choice.message)
] == [
(block["id"], block["name"], json.dumps(block["input"]))
for block in content
if block["type"] == "server_tool_use"
]
if not expected_tool_call_ids:
assert choice.message.content == _PARIS_TEXT
def _replayed(*messages: object):
return anthropic_messages_pt(
messages=cast(list[AllMessageValues], list(messages)), # cast-ok: replays history the way clients send it back
model="claude-opus-5-5",
llm_provider="anthropic",
)
def test_web_search_answer_round_trips_without_any_client_tool_call():
message: Final = _parsed_assistant_message(_web_search_turn(_PARIS_ANSWER), "end_turn")
replayed: Final = _replayed(
{"role": "user", "content": "Tell me about Paris"},
message.model_dump(),
{"role": "user", "content": "And Lyon?"},
)
assert _block_ids(replayed[1], "server_tool_use", "id") == ["srvtoolu_01Web"]
assert _block_ids(replayed[1], "web_search_tool_result", "tool_use_id") == ["srvtoolu_01Web"]
assert [
message_field(block, "text")
for block in parts_of(replayed[1]["content"])
if message_field(block, "type") == "text"
] == [_PARIS_TEXT]
def _block_ids(message: Mapping[str, object], block_type: str, id_key: str) -> list[str]:
return sorted(
block_id
for block in parts_of(message["content"])
if message_field(block, "type") == block_type and isinstance(block_id := message_field(block, id_key), str)
)
def _as_returned_before_server_tool_calls_moved(message: Message) -> Mapping[str, object]:
dumped: Final[Mapping[str, object]] = message.model_dump()
provider_fields: Final = validated_object_mapping(dumped.get("provider_specific_fields")) or {}
return {
**dumped,
"tool_calls": [*anthropic_server_tool_call_params(message), *parts_of(dumped.get("tool_calls"))],
"provider_specific_fields": {
key: value for key, value in provider_fields.items() if key != "server_tool_calls"
},
}
_SEARCH_THAT_FOUND_NOTHING: Final = _tool_search_turn(
@ -3806,21 +3958,33 @@ def test_replayed_search_that_found_nothing_answers_every_tool_use_and_ends_on_a
server_tool_use_ids: list[str],
monkeypatch: pytest.MonkeyPatch,
):
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
message: Final = _parsed_assistant_message(_SEARCH_THAT_FOUND_NOTHING, "end_turn")
monkeypatch.setattr(litellm, "modify_params", modify_params)
replayed: Final = anthropic_messages_pt(
messages=[{"role": "user", "content": "Find a tool and use it"}, message.model_dump(), *follow_ups],
model="claude-opus-5-5",
llm_provider="anthropic",
replayed: Final = _replayed(
{"role": "user", "content": "Find a tool and use it"},
_as_returned_before_server_tool_calls_moved(message),
*follow_ups,
)
assert [turn["role"] for turn in replayed] == ["user", "assistant", "user"]
assert _block_ids(replayed[2], "tool_result", "tool_use_id") == _block_ids(replayed[1], "tool_use", "id")
assert _block_ids(replayed[1], "server_tool_use", "id") == server_tool_use_ids
def test_replayed_search_that_found_nothing_keeps_its_result_when_the_call_never_reached_the_client(
monkeypatch: pytest.MonkeyPatch,
):
message: Final = _parsed_assistant_message(_SEARCH_THAT_FOUND_NOTHING, "end_turn")
monkeypatch.setattr(litellm, "modify_params", True)
replayed: Final = _replayed({"role": "user", "content": "Find a tool and use it"}, message.model_dump())
assert [turn["role"] for turn in replayed] == ["user", "assistant"]
assert _block_ids(replayed[1], "server_tool_use", "id") == ["srvtoolu_01Search"]
assert _block_ids(replayed[1], "tool_search_tool_result", "tool_use_id") == ["srvtoolu_01Search"]
assert _block_ids(replayed[1], "tool_use", "id") == []
_CODE_EXECUTION_BLOCKS: Final = [
{
"type": "server_tool_use",

53
tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py
View file

@ -81,6 +81,59 @@ def test_native_compaction_response_roundtrip(text: str | None, count: int, expe
assert result["stop_reason"] == expected_stop
_SERVER_AND_CLIENT_TOOL_TURN: Final[tuple[dict[str, object], ...]] = (
{
"type": "server_tool_use",
"id": "srvtoolu_01Fetch",
"name": "web_fetch",
"input": {"url": "https://example.com/"},
},
{
"type": "web_fetch_tool_result",
"tool_use_id": "srvtoolu_01Fetch",
"content": {"type": "web_fetch_result", "url": "https://example.com/"},
},
{"type": "server_tool_use", "id": "srvtoolu_02Search", "name": "web_search", "input": {"query": "Paris"}},
{"type": "text", "text": "The page is titled Example."},
{"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}},
)
def test_claude_server_tool_blocks_survive_the_round_trip_through_chat_completions() -> None:
from unittest.mock import MagicMock
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
anthropic_response: Final = {
"id": "msg_01Fetch",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": list(_SERVER_AND_CLIENT_TOOL_TURN),
"stop_reason": "tool_use",
"stop_sequence": None,
"usage": {"input_tokens": 100, "output_tokens": 50},
}
raw_response: Final = MagicMock(spec=httpx.Response)
raw_response.json.return_value = anthropic_response
raw_response.status_code = 200
raw_response.headers = {}
chat_response: Final = AnthropicConfig().transform_parsed_response(
completion_response=anthropic_response,
raw_response=raw_response,
model_response=ModelResponse(),
json_mode=False,
prefix_prompt=None,
)
result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(chat_response)
assert [(block["type"], block.get("id") or block.get("tool_use_id")) for block in result["content"]] == [
(block["type"], block.get("id") or block.get("tool_use_id")) for block in _SERVER_AND_CLIENT_TOOL_TURN
]
assert result["stop_reason"] == "tool_use"
def test_translate_chat_refusal_to_anthropic_response():
response = ModelResponse(
id="chatcmpl-refusal",

323
tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py
View file

@ -7,7 +7,7 @@ with guardrail transformations, including tool calls.
import json
from collections.abc import Mapping
from typing import Any, Literal, Optional
from typing import Any, Final, Literal, Optional
import pytest
@ -17,6 +17,7 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import Streami
from litellm.llms.openai.chat.guardrail_translation.handler import (
OpenAIChatCompletionsHandler,
)
from litellm.types.responses.main import build_web_search_call
from litellm.types.utils import (
ChatCompletionMessageToolCall,
Choices,
@ -144,6 +145,53 @@ class MockMisalignedToolCallGuardrail(CustomGuardrail):
)
def _server_tool_call(call_id: str, name: str, arguments: Mapping[str, object]) -> dict[str, object]:
return {"id": call_id, "type": "function", "function": {"name": name, "arguments": json.dumps(arguments)}}
def _tool_call_ids_names_and_arguments(tool_calls) -> list[tuple[str, str, str]]:
return [
(tool_call["id"], tool_call["function"]["name"], tool_call["function"]["arguments"])
for tool_call in tool_calls
]
def _hosted_web_call_inputs(web_search_calls) -> list[tuple[str, str, Optional[str]]]:
return [
(call.id, call.status, call.action.query if call.action.type == "search" else call.action.url)
for call in web_search_calls
]
_SEARCH_INPUT: Final = {"query": "jane@example.com"}
_FETCH_INPUT: Final = {"url": "https://example.com/?email=jane@example.com"}
class QueryRedactingGuardrail(CustomGuardrail):
"""Returns copied tool calls with any ``query`` or ``url`` argument redacted, leaving other calls as they came."""
async def apply_guardrail(
self,
inputs: GenericGuardrailAPIInputs,
request_data: dict,
input_type: Literal["request", "response"],
logging_obj: Optional[Any] = None,
) -> GenericGuardrailAPIInputs:
def redacted(tool_call):
arguments = json.loads(tool_call["function"]["arguments"])
redacted_arguments = {
key: "[REDACTED]" if key in ("query", "url") else value for key, value in arguments.items()
}
if redacted_arguments == arguments:
return tool_call
return {**tool_call, "function": {**tool_call["function"], "arguments": json.dumps(redacted_arguments)}}
return GenericGuardrailAPIInputs(
texts=inputs.get("texts", []),
tool_calls=[redacted(tool_call) for tool_call in inputs.get("tool_calls", [])],
)
class TestOpenAIChatCompletionsHandlerToolsInput:
"""Test input processing with tools (function definitions)"""
@ -906,6 +954,132 @@ class TestOpenAIChatCompletionsHandlerToolCallsOutput:
assert tool_calls[0].function.arguments == first_args
assert tool_calls[1].function.arguments == second_args
@pytest.mark.asyncio
async def test_output_guardrail_sees_a_response_that_only_carries_server_tool_calls(self):
handler = OpenAIChatCompletionsHandler()
guardrail = TestOpenAIChatCompletionsHandlerStreamingOutput._recording_guardrail()
response = ModelResponse(
model="claude-sonnet-4-5",
choices=[
Choices(
finish_reason="stop",
index=0,
message=Message(
content=None,
role="assistant",
provider_specific_fields={
"server_tool_calls": [_server_tool_call("srvtoolu_01Search", "web_search", {"query": "x"})]
},
),
)
],
)
await handler.process_output_response(response, guardrail)
assert _tool_call_ids_names_and_arguments(guardrail.seen_inputs[-1]["tool_calls"]) == [
("srvtoolu_01Search", "web_search", '{"query": "x"}')
]
@pytest.mark.asyncio
async def test_output_guardrail_rewrite_of_a_server_tool_call_lands_in_server_tool_calls_only(self):
handler = OpenAIChatCompletionsHandler()
weather_args = json.dumps({"city": "Paris", "query": "forecast"})
response = ModelResponse(
model="claude-sonnet-4-5",
choices=[
Choices(
finish_reason="tool_calls",
index=0,
message=Message(
content="Looking it up",
role="assistant",
tool_calls=[
ChatCompletionMessageToolCall(
id="call_1",
type="function",
function=Function(name="get_weather", arguments=json.dumps({"city": "Paris"})),
)
],
provider_specific_fields={
"server_tool_calls": [
_server_tool_call("srvtoolu_01Search", "web_search", {"query": "jane@example.com"})
]
},
),
),
Choices(
finish_reason="tool_calls",
index=1,
message=Message(
content=None,
role="assistant",
tool_calls=[
ChatCompletionMessageToolCall(
id="call_2",
type="function",
function=Function(name="get_weather", arguments=weather_args),
)
],
),
),
],
)
await handler.process_output_response(response, QueryRedactingGuardrail(guardrail_name="redact"))
first, second = response.choices
assert _tool_call_ids_names_and_arguments(first.message.provider_specific_fields["server_tool_calls"]) == [
("srvtoolu_01Search", "web_search", '{"query": "[REDACTED]"}')
]
assert [(call.id, call.function.arguments) for call in first.message.tool_calls] == [
("call_1", '{"city": "Paris"}')
]
assert [(call.id, call.function.arguments) for call in second.message.tool_calls] == [
("call_2", '{"city": "Paris", "query": "[REDACTED]"}')
]
assert not (second.message.provider_specific_fields or {}).get("server_tool_calls")
@pytest.mark.asyncio
async def test_output_guardrail_rewrite_of_a_server_web_call_reaches_its_hosted_copy(self):
handler = OpenAIChatCompletionsHandler()
response = ModelResponse(
model="claude-sonnet-4-5",
choices=[
Choices(
finish_reason="stop",
index=0,
message=Message(
content=None,
role="assistant",
provider_specific_fields={
"server_tool_calls": [
_server_tool_call("srvtoolu_01Search", "web_search", _SEARCH_INPUT),
_server_tool_call("srvtoolu_01Fetch", "web_fetch", _FETCH_INPUT),
],
"web_search_calls": [
build_web_search_call("srvtoolu_01Search", _SEARCH_INPUT, {"content": []}),
build_web_search_call(
"srvtoolu_01Fetch",
_FETCH_INPUT,
{"content": []},
status="in_progress",
tool_name="web_fetch",
),
],
},
),
)
],
)
await handler.process_output_response(response, QueryRedactingGuardrail(guardrail_name="redact"))
assert _hosted_web_call_inputs(response.choices[0].message.provider_specific_fields["web_search_calls"]) == [
("ws_srvtoolu_01Search", "completed", "[REDACTED]"),
("ws_srvtoolu_01Fetch", "in_progress", "[REDACTED]"),
]
class MockPassThroughGuardrail(CustomGuardrail):
"""Mock guardrail that passes through without blocking - for testing streaming fallback behavior"""
@ -1462,6 +1636,130 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
assert chunks[0].choices[0].delta.content == "hello [MASKED]"
assert chunks[1].choices[0].delta.content in (None, "")
@staticmethod
def _ended_server_tool_call_stream_chunks() -> list:
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
def chunk(delta: Delta, finish_reason: Optional[str] = None) -> ModelResponseStream:
return ModelResponseStream(
id="chatcmpl-123",
created=1234567890,
model="claude-sonnet-4-5",
object="chat.completion.chunk",
choices=[StreamingChoices(index=0, delta=delta, finish_reason=finish_reason)],
)
search = {**_server_tool_call("srvtoolu_01Search", "web_search", {"query": "jane@example.com"}), "index": 0}
return [
chunk(Delta(content="Searching")),
chunk(Delta(content=None, provider_specific_fields={"server_tool_calls": [search]})),
chunk(Delta(content=None), finish_reason="stop"),
]
@staticmethod
def _ended_server_fetch_stream_chunks() -> list:
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
def chunk(delta: Delta, finish_reason: Optional[str] = None) -> ModelResponseStream:
return ModelResponseStream(
id="chatcmpl-123",
created=1234567890,
model="claude-sonnet-4-5",
object="chat.completion.chunk",
choices=[StreamingChoices(index=0, delta=delta, finish_reason=finish_reason)],
)
def hosted_fetch(tool_input: dict, result: dict, status: Optional[str]) -> dict:
return {
"srvtoolu_01Fetch": build_web_search_call(
"srvtoolu_01Fetch", tool_input, result, status=status, tool_name="web_fetch"
)
}
fetch = {**_server_tool_call("srvtoolu_01Fetch", "web_fetch", _FETCH_INPUT), "index": 0}
return [
chunk(
Delta(
content=None,
provider_specific_fields={"web_search_calls": hosted_fetch({}, {"content": []}, "in_progress")},
)
),
chunk(
Delta(
content=None,
provider_specific_fields={
"server_tool_calls": [fetch],
"web_search_calls": hosted_fetch(_FETCH_INPUT, {"content": []}, "in_progress"),
},
)
),
chunk(
Delta(
content=None,
provider_specific_fields={
"web_search_calls": hosted_fetch(_FETCH_INPUT, {"content": {"type": "web_fetch_result"}}, None)
},
)
),
chunk(Delta(content="Fetched"), finish_reason="stop"),
]
@pytest.mark.asyncio
async def test_deliver_ended_stream_rewrites_reach_every_hosted_copy_of_a_server_web_call(self):
handler = OpenAIChatCompletionsHandler()
chunks = self._ended_server_fetch_stream_chunks()
await handler.process_output_streaming_response(
responses_so_far=chunks,
guardrail_to_apply=QueryRedactingGuardrail(guardrail_name="redact"),
litellm_logging_obj=None,
deliver_ended_stream_rewrites=True,
)
assert [
_hosted_web_call_inputs(chunk.choices[0].delta.provider_specific_fields["web_search_calls"].values())
for chunk in chunks[:3]
] == [
[("ws_srvtoolu_01Fetch", "in_progress", "[REDACTED]")],
[("ws_srvtoolu_01Fetch", "in_progress", "[REDACTED]")],
[("ws_srvtoolu_01Fetch", "completed", "[REDACTED]")],
]
@pytest.mark.asyncio
async def test_ended_stream_guardrail_sees_streamed_server_tool_calls(self):
handler = OpenAIChatCompletionsHandler()
guardrail = self._recording_guardrail()
await handler.process_output_streaming_response(
responses_so_far=self._ended_server_tool_call_stream_chunks(),
guardrail_to_apply=guardrail,
litellm_logging_obj=None,
deliver_ended_stream_rewrites=True,
)
assert _tool_call_ids_names_and_arguments(guardrail.seen_inputs[-1]["tool_calls"]) == [
("srvtoolu_01Search", "web_search", '{"query": "jane@example.com"}')
]
@pytest.mark.asyncio
async def test_deliver_ended_stream_rewrites_writes_server_tool_call_arguments_back_into_chunks(self):
handler = OpenAIChatCompletionsHandler()
chunks = self._ended_server_tool_call_stream_chunks()
await handler.process_output_streaming_response(
responses_so_far=chunks,
guardrail_to_apply=QueryRedactingGuardrail(guardrail_name="redact"),
litellm_logging_obj=None,
deliver_ended_stream_rewrites=True,
)
delta = chunks[1].choices[0].delta
assert _tool_call_ids_names_and_arguments(delta.provider_specific_fields["server_tool_calls"]) == [
("srvtoolu_01Search", "web_search", '{"query": "[REDACTED]"}')
]
assert delta.tool_calls is None
assert chunks[0].choices[0].delta.content == "Searching"
class TestUndecoratedGuardrailIsRecorded:
"""LIT-5983 regression: the handler calls apply_guardrail bare, so a custom guardrail
@ -2323,6 +2621,29 @@ class TestStreamingScanKey:
assert len(ended_key.tool_calls) == 1 and "run_shell" in ended_key.tool_calls[0]
assert ended_key.tool_calls_in_flight is False
def test_server_tool_calls_hold_the_window_and_enter_the_key_once_the_stream_has_ended(self):
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
handler = OpenAIChatCompletionsHandler()
server_chunk = ModelResponseStream(
choices=[
StreamingChoices(
index=0,
delta=Delta(
content=None,
provider_specific_fields={
"server_tool_calls": [_server_tool_call("srvtoolu_01Search", "web_search", {"query": "x"})]
},
),
)
]
)
open_key = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk])
ended_key = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk, self._chunk(None, "stop")])
assert open_key.tool_calls_in_flight is True
assert len(ended_key.tool_calls) == 1 and "web_search" in ended_key.tool_calls[0]
assert ended_key != open_key
def test_text_after_the_first_choice_finishes_still_changes_the_key(self):
handler = OpenAIChatCompletionsHandler()
first_done = [self._chunk("a", index=0), self._chunk("b", finish_reason="stop", index=0)]

159
tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py
View file

@ -20,7 +20,9 @@ import pytest
from fastapi import HTTPException
from pydantic import BaseModel
from openai.types.responses.response_function_web_search import ActionSearch
from openai.types.responses import (
ResponseFunctionWebSearch,
ResponseCustomToolCall,
ResponseCustomToolCallInputDeltaEvent,
ResponseCustomToolCallInputDoneEvent,
@ -48,6 +50,7 @@ from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.responses.litellm_completion_transformation.tool_search import build_tool_search_call
from litellm.types.responses.main import (
CustomToolCallOutputItem,
build_web_search_call,
GenericResponseOutputItem,
OutputText,
OutputToolSearchCall,
@ -237,6 +240,30 @@ CUSTOM_TOOL_CALL_ITEM = {
}
def _claude_web_calls_and_a_native_search() -> list[ResponseFunctionWebSearch]:
return [
build_web_search_call("srvtoolu_1", {"query": "persimmon"}, None, "completed"),
build_web_search_call("srvtoolu_2", {"url": "https://persimmon.example"}, None, "in_progress", "web_fetch"),
ResponseFunctionWebSearch(
id="ws_native",
type="web_search_call",
status="completed",
action=ActionSearch(type="search", query="persimmon"),
),
]
def _web_call_actions(items: list[object]) -> list[dict[str, object]]:
return [ResponseFunctionWebSearch.model_validate(item, from_attributes=True).action.model_dump() for item in items]
_MASKED_WEB_CALL_ACTIONS: Final = [
{"type": "search", "query": "[MASKED]", "queries": ["[MASKED]"], "sources": []},
{"type": "open_page", "url": "https://[MASKED].example"},
{"type": "search", "query": "persimmon", "queries": None, "sources": None},
]
class TestOpenAIResponsesHandlerDiscovery:
"""Test that the handler is properly discovered by the guardrail system"""
@ -1810,6 +1837,27 @@ class TestOpenAIResponsesHandlerToolCallExtraction:
assert search_call.call_id == "srvtoolu_1"
assert "name" not in search_call.model_dump()
@pytest.mark.asyncio
async def test_process_output_response_scans_and_rewrites_claude_web_calls_but_not_native_search(self):
guardrail = RecordingPersimmonMaskingGuardrail(guardrail_name="mask")
response = ResponsesAPIResponse(
id="resp_1",
created_at=1,
model="claude-sonnet-5-5",
object="response",
status="completed",
output=_claude_web_calls_and_a_native_search(),
)
result = await OpenAIResponsesHandler().process_output_response(response, guardrail)
assert guardrail.seen_tool_calls == [
("web_search", '{"query": "persimmon"}'),
("web_fetch", '{"url": "https://persimmon.example"}'),
]
assert [item.type for item in result.output] == ["web_search_call"] * 3
assert _web_call_actions(result.output) == _MASKED_WEB_CALL_ACTIONS
@pytest.mark.asyncio
async def test_process_output_response_never_returns_original_search_arguments_after_a_non_json_rewrite(self):
response = ResponsesAPIResponse(
@ -2745,6 +2793,117 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing:
assert completed.response.output[0].arguments == {"query": "[MASKED]"}
assert "name" not in done.item.model_dump()
@pytest.mark.asyncio
async def test_deliver_ended_stream_scans_and_rewrites_claude_web_calls_but_not_native_search(self):
from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject
from litellm.types.llms.openai import (
OutputItemAddedEvent,
OutputItemDoneEvent,
ResponseCompletedEvent,
ResponsesAPIStreamEvents,
)
items = _claude_web_calls_and_a_native_search()
added = [
OutputItemAddedEvent(
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
output_index=index,
item=BaseLiteLLMOpenAIResponseObject(**{**item.model_dump(), "action": None}),
)
for index, item in enumerate(items)
]
done = [
OutputItemDoneEvent(
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
output_index=index,
sequence_number=index + 3,
item=BaseLiteLLMOpenAIResponseObject(**item.model_dump()),
)
for index, item in enumerate(items)
]
completed = ResponseCompletedEvent(
type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
response=ResponsesAPIResponse(
id="resp_1",
created_at=1,
model="claude-sonnet-5-5",
object="response",
status="completed",
output=_claude_web_calls_and_a_native_search(),
),
)
guardrail = RecordingPersimmonMaskingGuardrail(guardrail_name="mask")
await OpenAIResponsesHandler().process_output_streaming_response(
responses_so_far=[*added, *done, completed],
guardrail_to_apply=guardrail,
litellm_logging_obj=None,
deliver_ended_stream_rewrites=True,
)
assert sorted(guardrail.seen_tool_calls) == [
("web_fetch", '{"url": "https://persimmon.example"}'),
("web_search", '{"query": "persimmon"}'),
]
assert [event.item.model_dump()["action"] for event in added] == [None, None, None]
assert _web_call_actions([event.item for event in done]) == _MASKED_WEB_CALL_ACTIONS
assert _web_call_actions(completed.response.output) == _MASKED_WEB_CALL_ACTIONS
@pytest.mark.asyncio
async def test_deliver_ended_stream_rewrites_dict_shaped_claude_web_calls_but_not_native_search(self):
items = [item.model_dump() for item in _claude_web_calls_and_a_native_search()]
done = [
{"type": "response.output_item.done", "output_index": index, "item": dict(item)}
for index, item in enumerate(items)
]
completed = {
"type": "response.completed",
"response": {
"id": "resp_1",
"model": "claude-sonnet-5-5",
"status": "completed",
"output": [dict(item) for item in items],
},
}
guardrail = RecordingPersimmonMaskingGuardrail(guardrail_name="mask")
await OpenAIResponsesHandler().process_output_streaming_response(
responses_so_far=[*done, completed],
guardrail_to_apply=guardrail,
litellm_logging_obj=None,
deliver_ended_stream_rewrites=True,
)
assert sorted(guardrail.seen_tool_calls) == [
("web_fetch", '{"url": "https://persimmon.example"}'),
("web_search", '{"query": "persimmon"}'),
]
assert _web_call_actions([event["item"] for event in done]) == _MASKED_WEB_CALL_ACTIONS
assert _web_call_actions(completed["response"]["output"]) == _MASKED_WEB_CALL_ACTIONS
@pytest.mark.asyncio
async def test_streaming_output_item_done_reports_a_claude_web_search_call(self):
from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject
from litellm.types.llms.openai import OutputItemDoneEvent, ResponsesAPIStreamEvents
done = OutputItemDoneEvent(
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
output_index=0,
sequence_number=1,
item=BaseLiteLLMOpenAIResponseObject(
**build_web_search_call("srvtoolu_1", {"query": "persimmon"}, None, "completed").model_dump()
),
)
guardrail = RecordingPersimmonMaskingGuardrail(guardrail_name="mask")
await OpenAIResponsesHandler().process_output_streaming_response(
responses_so_far=[done],
guardrail_to_apply=guardrail,
litellm_logging_obj=None,
)
assert guardrail.seen_tool_calls == [("web_search", '{"query": "persimmon"}')]
@pytest.mark.asyncio
async def test_deliver_ended_stream_rewrites_a_native_tool_search_call_that_has_no_call_id(self):
def search_call(arguments: dict, status: str) -> dict:

31
tests/unit/proxy/db/test_spend_log_tool_index.py
View file

@ -23,6 +23,7 @@ from litellm.proxy.db.spend_log_tool_index import (
)
from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import ModelResponse
from tests.unit.proxy.db.fake_prisma_engine import engine_call
@ -204,6 +205,36 @@ class TestResponseToolCallNames:
def test_blank_names_are_dropped(self):
assert response_tool_call_names(_response_with_tool_calls(" ", "real_tool")) == ("real_tool",)
@pytest.mark.parametrize("as_object", [False, True], ids=["dict", "model_response"])
def test_server_tool_calls_of_every_choice_are_named_before_client_tool_calls(self, as_object: bool):
def tool_call(call_id: str, name: str) -> dict[str, object]:
return {"id": call_id, "type": "function", "function": {"name": name, "arguments": "{}"}}
response: dict[str, object] = {
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": None,
"tool_calls": [tool_call("call_1", "get_weather")],
"provider_specific_fields": {"server_tool_calls": [tool_call("srvtoolu_01", "web_search")]},
},
},
{
"index": 1,
"message": {
"role": "assistant",
"content": "done",
"provider_specific_fields": {"server_tool_calls": [tool_call("srvtoolu_02", "web_fetch")]},
},
},
]
}
completion_response = ModelResponse(**response) if as_object else response
assert response_tool_call_names(completion_response) == ("web_search", "web_fetch", "get_weather")
def test_responses_api_output_function_calls(self):
# Regression: /v1/responses carries invocations in output[] items of
# type function_call, not in choices; they must reach the rollup.

77
tests/unit/proxy/guardrails/guardrail_hooks/test_cato_networks.py
View file

@ -2237,6 +2237,83 @@ async def test_post_call_success_hook_redacts_both_content_and_tool_arguments():
assert message.tool_calls[0].function.arguments == '{"to": "[NAME_1]"}'
@pytest.mark.asyncio
async def test_post_call_success_hook_inspects_and_redacts_server_tool_call_arguments():
guard = _make_guardrail()
request_data = {"messages": [{"role": "user", "content": "look up persimmon"}]}
def side_effect(url, *args, **kwargs):
last = kwargs["json"]["messages"][-1]["content"]
return _make_response(
{
"analysis_result": {"policy_drill_down": {"PII": {}}},
"required_action": {"action_type": "anonymize_action", "policy_name": "PII"},
"redacted_chat": {
"all_redacted_messages": [
{"role": "user", "content": "look up persimmon"},
{"role": "assistant", "content": last.replace("persimmon", "[FRUIT_1]")},
]
},
}
)
llm_response = ModelResponse(
choices=[
{
"finish_reason": "tool_calls",
"index": 0,
"message": {
"content": None,
"role": "assistant",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city": "Paris"}'},
}
],
"provider_specific_fields": {
"server_tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "persimmon"}'},
}
]
},
},
}
]
)
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
side_effect=side_effect,
) as mock_post:
result = await guard.async_post_call_success_hook(
data=request_data,
response=llm_response,
user_api_key_dict=UserAPIKeyAuth(),
)
inspected = [call.kwargs["json"]["messages"][-1] for call in mock_post.call_args_list]
assert inspected == [
{"role": "assistant", "content": '{"query": "persimmon"}'},
{"role": "assistant", "content": '{"city": "Paris"}'},
]
message = result.choices[0].message
assert message.provider_specific_fields == {
"server_tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "[FRUIT_1]"}'},
}
]
}
assert message.content is None
assert message.tool_calls[0].function.arguments == '{"city": "Paris"}'
def _make_responses_api_response(output: list) -> ResponsesAPIResponse:
return ResponsesAPIResponse(id="resp-1", created_at=0, output=output)

134
tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py
View file

@ -404,6 +404,33 @@ class TestCiscoAIDefenseChatMode:
)
_CLAUDE_HOSTED_WEB_ACTIONS = (
{"type": "search", "query": "ssn 123-45-6789"},
{"type": "open_page", "url": "https://example.com/?ssn=123-45-6789"},
)
def _claude_hosted_web_call_response(action: dict):
from openai.types.responses import ResponseFunctionWebSearch
from litellm.types.llms.openai import ResponsesAPIResponse
return ResponsesAPIResponse(
id="resp_1",
created_at=0,
output=[
ResponseFunctionWebSearch(
id="ws_srvtoolu_01", type="web_search_call", status="completed", action=action
)
],
parallel_tool_calls=False,
tool_choice=None,
tools=None,
top_p=None,
usage=None,
)
class TestCiscoAIDefenseResponsesAPIOutput:
@staticmethod
@ -505,6 +532,23 @@ class TestCiscoAIDefenseResponsesAPIOutput:
joined = " ".join(m.get("content", "") for m in (sent.get("messages") or []))
assert "4111-1111-1111-1111" in joined
@pytest.mark.parametrize("action", _CLAUDE_HOSTED_WEB_ACTIONS)
@pytest.mark.asyncio
async def test_post_call_scans_claude_hosted_web_call(self, action):
g = _make_guardrail(event_hook="post_call")
post_mock = AsyncMock(return_value=_safe_response())
with _patch_inspection_post(g, post_mock):
await g.async_post_call_success_hook(
data={"input": [{"role": "user", "content": "look it up"}]},
user_api_key_dict=UserAPIKeyAuth(),
response=_claude_hosted_web_call_response(action),
)
sent = post_mock.call_args.kwargs["json"]
joined = " ".join(m.get("content", "") for m in (sent.get("messages") or []))
assert "123-45-6789" in joined
@pytest.mark.asyncio
async def test_post_call_responses_api_violation_is_blocked(self):
g = _make_guardrail(event_hook="post_call")
@ -573,6 +617,24 @@ class TestCiscoAIDefenseResponsesAPIOutputRedaction:
)
@pytest.mark.parametrize("action", _CLAUDE_HOSTED_WEB_ACTIONS)
@pytest.mark.asyncio
async def test_redact_clears_claude_hosted_web_call(self, action):
g = _make_guardrail(event_hook="post_call", on_flagged_action="monitor")
cisco_resp = _redact_response(sanitized_text="[REDACTED]")
with _patch_inspection_post(g, AsyncMock(return_value=cisco_resp)):
result = await g.async_post_call_success_hook(
data={"input": [{"role": "user", "content": "look it up"}]},
user_api_key_dict=UserAPIKeyAuth(),
response=_claude_hosted_web_call_response(action),
)
call = result.output[0]
assert (call.id, call.action.type) == ("ws_srvtoolu_01", action["type"])
assert "123-45-6789" not in call.action.model_dump_json()
class TestCiscoAIDefenseResponsesAPIInputRedaction:
@pytest.mark.parametrize(
@ -1777,6 +1839,78 @@ class TestCiscoAIDefenseToolCallBypass:
f"included in the scanned text. Sent: {sent!r}"
)
@staticmethod
def _server_tool_call_response() -> ModelResponse:
from litellm.types.utils import ChatCompletionMessageToolCall, Function
return ModelResponse(
choices=[
Choices(
index=0,
finish_reason="tool_calls",
message=Message(
role="assistant",
content=None,
tool_calls=[
ChatCompletionMessageToolCall(
id="call_1",
type="function",
function=Function(name="get_weather", arguments='{"city": "Paris"}'),
)
],
provider_specific_fields={
"server_tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "persimmon"}'},
}
]
},
),
)
]
)
@pytest.mark.asyncio
async def test_post_call_scan_sends_server_tool_call_arguments_to_cisco(self):
g = _make_guardrail(event_hook="post_call")
post_mock = AsyncMock(return_value=_safe_response())
with _patch_inspection_post(g, post_mock):
await g.async_post_call_success_hook(
data={"messages": [{"role": "user", "content": "look up persimmon"}]},
user_api_key_dict=UserAPIKeyAuth(),
response=self._server_tool_call_response(),
)
assert [call.kwargs["json"]["messages"] for call in post_mock.call_args_list] == [
[
{"role": "user", "content": "look up persimmon"},
{"role": "assistant", "content": '{"query": "persimmon"} {"city": "Paris"}'},
]
]
@pytest.mark.asyncio
async def test_post_call_redact_clears_server_tool_call_arguments(self):
g = _make_guardrail(event_hook="post_call", on_flagged_action="monitor")
cisco_resp = _redact_response(sanitized_text="[REDACTED]")
with _patch_inspection_post(g, AsyncMock(return_value=cisco_resp)):
result = await g.async_post_call_success_hook(
data={"messages": [{"role": "user", "content": "look up persimmon"}]},
user_api_key_dict=UserAPIKeyAuth(),
response=self._server_tool_call_response(),
)
message = result.choices[0].message
assert message.provider_specific_fields == {
"server_tool_calls": [
{"id": "srvtoolu_01", "type": "function", "function": {"name": "web_search", "arguments": "{}"}}
]
}
assert message.tool_calls[0].function.arguments == "{}"
class TestCiscoAIDefenseToolDefinitionBypass:

142
tests/unit/proxy/guardrails/guardrail_hooks/test_lasso.py
View file

@ -1,3 +1,4 @@
import json
import os
import uuid
from typing import Final
@ -1549,3 +1550,144 @@ async def test_api_error_handling_legacy_messages(
assert "Failed to verify request safety with Lasso API" in str(excinfo.value)
assert error_message in str(excinfo.value)
assert route.called
def _reply_with_web_search_call(
arguments: str, tool_calls: list[dict[str, object]] | None = None
) -> litellm.ModelResponse:
return litellm.ModelResponse(
choices=[
litellm.Choices(
index=0,
finish_reason="tool_calls",
message=litellm.Message(
role="assistant",
content=None,
tool_calls=tool_calls,
provider_specific_fields={
"server_tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": arguments},
}
]
},
),
)
]
)
@pytest.mark.asyncio
async def test_post_call_scan_sends_server_tool_calls_as_tool_use_entries(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
lasso_guardrail: Final = LassoGuardrail(
lasso_api_key="test-key",
user_id="test-user",
conversation_id="test-conversation",
guardrail_name="test-guard",
event_hook="post_call",
default_on=True,
)
route: Final = respx_mock.post("https://server.lasso.security/gateway/v3/classify").mock(
return_value=httpx.Response(200, json={"violations_detected": False, "deputies": {}, "findings": {}})
)
response: Final = _reply_with_web_search_call(
'{"query": "persimmon"}',
tool_calls=[
{
"id": "call_01",
"type": "function",
"function": {"name": "lookup_order", "arguments": '{"order_id": "A-17"}'},
}
],
)
await lasso_guardrail.async_post_call_success_hook(
data={"messages": [{"role": "user", "content": "search persimmons"}]},
user_api_key_dict=UserAPIKeyAuth(),
response=response,
)
assert json.loads(route.calls.last.request.content) == {
"messages": [
{
"role": "model",
"content": {
"type": "tool_use",
"id": "srvtoolu_01",
"name": "web_search",
"input": {"query": "persimmon"},
},
},
{
"role": "model",
"content": {
"type": "tool_use",
"id": "call_01",
"name": "lookup_order",
"input": {"order_id": "A-17"},
},
},
],
"messageType": "COMPLETION",
"source": {"type": "litellm"},
"userId": "test-user",
"sessionId": "test-conversation",
}
@pytest.mark.asyncio
async def test_post_call_masking_rewrites_server_tool_call_arguments(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
lasso_guardrail: Final = LassoGuardrail(
lasso_api_key="test-key",
user_id="test-user",
conversation_id="test-conversation",
mask=True,
guardrail_name="test-guard",
event_hook="post_call",
default_on=True,
)
respx_mock.post("https://server.lasso.security/gateway/v3/classifix").mock(
return_value=httpx.Response(
200,
json={
"violations_detected": True,
"deputies": {"pattern-detection": True},
"findings": {"pattern-detection": [{"action": "AUTO_MASKING", "severity": "HIGH"}]},
"messages": [
{
"role": "model",
"content": {
"type": "tool_use",
"id": "srvtoolu_01",
"name": "web_search",
"input": {"query": "<EMAIL_ADDRESS> persimmon"},
},
}
],
},
)
)
result: Final = await lasso_guardrail.async_post_call_success_hook(
data={"messages": [{"role": "user", "content": "search persimmons"}]},
user_api_key_dict=UserAPIKeyAuth(),
response=_reply_with_web_search_call('{"query": "jane@example.com persimmon"}'),
)
assert result.choices[0].message.provider_specific_fields == {
"server_tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "<EMAIL_ADDRESS> persimmon"}'},
}
]
}

49
tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py
View file

@ -893,6 +893,55 @@ class TestRestoration:
assert result.choices[0].message.content == "a@b.com"
@pytest.mark.asyncio
async def test_server_tool_call_arguments_are_restored(self):
guardrail, _ = _shielded({"[EMAIL_1]": "a@b.com", "[URL_1]": "https://b.com/a"})
response = ModelResponse(
choices=[
Choices(
index=0,
finish_reason="tool_calls",
message=Message(
role="assistant",
content=None,
provider_specific_fields={
"server_tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "[EMAIL_1] persimmon"}'},
},
{
"id": "srvtoolu_02",
"type": "function",
"function": {"name": "web_fetch", "arguments": '{"url": "[URL_1]"}'},
},
]
},
),
)
]
)
result = await guardrail.async_post_call_success_hook(
data={"messages": []}, user_api_key_dict=None, response=response
)
assert result.choices[0].message.provider_specific_fields == {
"server_tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "a@b.com persimmon"}'},
},
{
"id": "srvtoolu_02",
"type": "function",
"function": {"name": "web_fetch", "arguments": '{"url": "https://b.com/a"}'},
},
]
}
@pytest.mark.asyncio
async def test_responses_api_shape_is_restored(self):
"""The Responses API reply carries output items, not choices.

57
tests/unit/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py
View file

@ -2072,6 +2072,63 @@ class TestCompletionResponseTextPartsToolCalls:
sent_text = mock_check.call_args.kwargs["text"]
assert '{"credit_card": "4111-1111-1111-1111"}' in sent_text
@pytest.mark.asyncio
async def test_post_call_hook_sends_server_tool_call_args_to_purview(self):
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.utils import Choices, Message, ModelResponse
def graph(request: httpx.Request) -> httpx.Response:
if request.url.host == "login.microsoftonline.com":
return httpx.Response(200, json={"access_token": "graph-token", "expires_in": 3600})
if not request.url.path.endswith("/processContent"):
return httpx.Response(200, json={"value": []}, headers={"ETag": "scope-etag"})
return httpx.Response(200, json={"protectionScopeState": "unchanged", "policyActions": []})
transport: Final = Mock(side_effect=graph)
guardrail: Final = _make_guardrail()
guardrail.async_handler = AsyncHTTPHandler(transport=httpx.MockTransport(transport))
response: Final = ModelResponse(
choices=[
Choices(
index=0,
finish_reason="tool_calls",
message=Message(
role="assistant",
content=None,
tool_calls=[
{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city": "Paris"}'},
}
],
provider_specific_fields={
"server_tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "persimmon"}'},
}
]
},
),
)
],
)
await guardrail.async_post_call_success_hook(
data={},
user_api_key_dict=UserAPIKeyAuth(api_key="test", user_id="user-123"),
response=response,
)
scanned: Final = [
json.loads(call.args[0].content)["contentToProcess"]["contentEntries"][0]["content"]["data"]
for call in transport.call_args_list
if call.args[0].url.path.endswith("/processContent")
]
assert scanned == ['{"query": "persimmon"}\n\n---\n\n{"city": "Paris"}']
def test_responses_api_function_call_args_included(self):
"""Function-call arguments in ``ResponsesAPIResponse.output`` must be DLP-scanned."""
from litellm.types.llms.openai import ResponsesAPIResponse

66
tests/unit/proxy/guardrails/guardrail_hooks/test_pangea.py
View file

@ -1,5 +1,7 @@
from unittest.mock import AsyncMock, patch
from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch
import httpx
import pytest
from fastapi import HTTPException
@ -11,6 +13,7 @@ from litellm.proxy.guardrails.guardrail_hooks.pangea.pangea import (
)
from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2
from litellm.types.guardrails import GuardrailEventHooks, LitellmParams
from litellm.types.responses.main import build_web_search_call
from litellm.types.utils import Choices, Message, ModelResponse
@ -368,3 +371,66 @@ async def test_pangea_ai_guard_response_transformed(pangea_guardrail):
)
assert response.choices[0]["message"]["content"] == "Yes, here is an SSN: <US_SSN>"
def _claude_fetch_call(url):
return {
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_fetch", "arguments": '{"url": "%s"}' % url},
}
def _with_masked_tool_calls(choice):
message = choice["message"]
masked_calls = [_claude_fetch_call("<URL>") for _ in message["tool_calls"]]
return {**choice, "message": {**message, "tool_calls": masked_calls}}
@pytest.mark.asyncio
async def test_pangea_ai_guard_response_masks_claude_server_tool_call_inputs(pangea_guardrail):
data = {"messages": [{"role": "user", "content": "Read https://example.com/"}]}
fetch = build_web_search_call("srvtoolu_01", {"url": "https://example.com/"}, None, tool_name="web_fetch")
sent_choices = []
async def _mask_tool_call_urls(_handler, **kwargs):
choices = kwargs["json"]["input"]["choices"]
sent_choices.extend(choices)
return httpx.Response(
status_code=200,
json={
"result": {
"blocked": False,
"transformed": True,
"output": {"choices": [_with_masked_tool_calls(choice) for choice in choices]},
}
},
request=httpx.Request(method="POST", url=kwargs["url"]),
)
with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new=_mask_tool_call_urls):
response = await pangea_guardrail.async_post_call_success_hook(
data=data,
user_api_key_dict=None,
response=ModelResponse(
choices=[
Choices(
message=Message(
role="assistant",
content="Done",
provider_specific_fields={
"server_tool_calls": [_claude_fetch_call("https://example.com/")],
"web_search_calls": [fetch],
},
)
)
]
),
)
assert sent_choices[0]["message"]["tool_calls"] == [_claude_fetch_call("https://example.com/")]
message = response.choices[0]["message"]
assert message["tool_calls"] is None
assert message["provider_specific_fields"]["server_tool_calls"] == [_claude_fetch_call("<URL>")]
hosted_copy = ResponseFunctionWebSearch.model_validate(message["provider_specific_fields"]["web_search_calls"][0])
assert hosted_copy.action.url == "<URL>"

81
tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py
View file

@ -6513,5 +6513,86 @@ class TestPanwAirsToolCallBlockMaskedDataRouting:
assert error["scan_id"] == "scan-tool-2"
class TestPanwAirsServerToolCalls:
SERVER_TOOL_CALL: Final = {
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "persimmon"}'},
}
@classmethod
def _response(cls, *client_tool_calls: ChatCompletionMessageToolCall) -> ModelResponse:
return ModelResponse(
choices=[
Choices(
index=0,
finish_reason="tool_calls",
message=Message(
role="assistant",
content=None,
tool_calls=list(client_tool_calls) or None,
provider_specific_fields={"server_tool_calls": [copy.deepcopy(cls.SERVER_TOOL_CALL)]},
),
)
],
model="claude-opus-4-1",
)
@staticmethod
def _airs(scan_result: Mapping[str, object]) -> tuple[PanwPrismaAirsHandler, MagicMock]:
transport: Final = MagicMock(side_effect=lambda request: httpx.Response(200, json=scan_result))
handler: Final = make_handler(
mask_response_content=True,
http_client=AsyncHTTPHandler(transport=httpx.MockTransport(transport)),
)
return handler, transport
@pytest.mark.asyncio
async def test_post_call_scan_sends_server_tool_call_query_to_airs(self, user_api_key_dict):
handler, transport = self._airs({"action": "allow", "category": "benign", "scan_id": "s1"})
response: Final = self._response(
ChatCompletionMessageToolCall(
id="call_1",
type="function",
function=Function(name="get_weather", arguments='{"city": "Paris"}'),
)
)
await handler.async_post_call_success_hook(
data={"model": "claude-opus-4-1", "litellm_call_id": "call-srv", "metadata": {}},
user_api_key_dict=user_api_key_dict,
response=response,
)
sent_contents: Final = [json.loads(call.args[0].content)["contents"] for call in transport.call_args_list]
assert sent_contents == [[{"response": '{"query": "persimmon"} {"city": "Paris"}'}]]
@pytest.mark.asyncio
async def test_post_call_masking_rewrites_server_tool_call_arguments(self, user_api_key_dict):
masked: Final = '{"query": "*********"}'
handler, _ = self._airs(
{
"action": "block",
"category": "sensitive_data",
"scan_id": "s1",
"response_masked_data": {"data": masked},
}
)
result: Final = await handler.async_post_call_success_hook(
data={"model": "claude-opus-4-1", "litellm_call_id": "call-srv-mask", "metadata": {}},
user_api_key_dict=user_api_key_dict,
response=self._response(),
)
message: Final = result.choices[0].message
assert message.provider_specific_fields == {
"server_tool_calls": [
{"id": "srvtoolu_01", "type": "function", "function": {"name": "web_search", "arguments": masked}}
]
}
assert message.tool_calls is None
if __name__ == "__main__":
pytest.main([__file__, "-v"])

42
tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py
View file

@ -1781,6 +1781,48 @@ async def test_pii_tokens_in_metadata_used_for_unmasking():
assert response.choices[0].message.content == "Hello John, how can I help you?"
@pytest.mark.asyncio
async def test_output_unmasking_restores_server_tool_call_arguments(mock_user_api_key):
guardrail: Final = OPTIONAL_PresidioPIIMasking(mock_testing=True, output_parse_pii=True)
response: Final = ModelResponse(
choices=[
Choices(
index=0,
finish_reason="tool_calls",
message=Message(
role="assistant",
content=None,
provider_specific_fields={
"server_tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "<PERSON_1> persimmon"}'},
}
]
},
),
)
]
)
result: Final = await guardrail.async_post_call_success_hook(
data={"metadata": {"pii_tokens": {"<PERSON_1>": "John Smith"}}},
user_api_key_dict=mock_user_api_key,
response=response,
)
assert result.choices[0].message.provider_specific_fields == {
"server_tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "John Smith persimmon"}'},
}
]
}
@pytest.mark.parametrize(
"initial_hook",
["pre_call", "during_call", "pre_mcp_call"],

53
tests/unit/proxy/guardrails/guardrail_hooks/test_repelloai.py
View file

@ -852,6 +852,59 @@ class TestRepelloAIPostCall:
in captured["json"]["scan_data"]["response"]
)
@pytest.mark.asyncio
async def test_server_tool_call_arguments_scanned(self, monkeypatch):
guardrail = _guardrail(event_hook="post_call")
response = ModelResponse(
choices=[
Choices(
index=0,
finish_reason="tool_calls",
message=Message(
role="assistant",
content=None,
tool_calls=[
{
"id": "call_01",
"type": "function",
"function": {
"name": "lookup_order",
"arguments": '{"order_id": "A-17"}',
},
}
],
provider_specific_fields={
"server_tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {
"name": "web_search",
"arguments": '{"query": "persimmon"}',
},
}
]
},
),
)
]
)
captured = {}
async def capture(url, headers, json, **_kwargs):
captured["json"] = json
return _verdict_response("passed", url)
monkeypatch.setattr(guardrail.async_handler, "post", capture)
await guardrail.async_post_call_success_hook(
data={"messages": [{"role": "user", "content": "q"}]},
user_api_key_dict=UserAPIKeyAuth(),
response=response,
)
assert captured["json"]["scan_data"] == {
"response": '{"query": "persimmon"}\n{"order_id": "A-17"}'
}
# ----------------------------------------------------------------------
# verdict handling: unknown / malformed responses must not fail open

73
tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py
View file

@ -39,6 +39,14 @@ from litellm.types.utils import (
)
_SERVER_WEB_SEARCH_CALL: Final = {
"id": "srvtoolu_01Search",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "weather in Paris"}'},
}
_ALLOWED_BASH_CALL: Final = {"id": "toolu_01Bash", "type": "function", "function": {"name": "Bash", "arguments": "{}"}}
class TestToolPermissionGuardrail:
"""Test class for Tool Permission Guardrail functionality"""
@ -1034,6 +1042,70 @@ class TestToolPermissionGuardrail:
assert "Permission denied" in choice.message.content
def _server_search_response(self) -> ModelResponse:
return ModelResponse(
choices=[
Choices(
finish_reason="tool_calls",
message={
"content": "Searching.",
"tool_calls": [_ALLOWED_BASH_CALL],
"provider_specific_fields": {"server_tool_calls": [_SERVER_WEB_SEARCH_CALL]},
},
)
]
)
@pytest.mark.asyncio
async def test_denied_server_tool_call_is_blocked(self):
with patch.object(self.guardrail, "should_run_guardrail", return_value=True):
with pytest.raises(GuardrailRaisedException):
await self.guardrail.async_post_call_success_hook(
data={}, user_api_key_dict=UserAPIKeyAuth(), response=self._server_search_response()
)
@pytest.mark.asyncio
async def test_denied_server_tool_call_in_a_chat_stream_is_blocked(self):
async def _stream():
yield ModelResponseStream(
choices=[
StreamingChoices(
index=0,
delta=Delta(
content="Searching.",
provider_specific_fields={"server_tool_calls": [{**_SERVER_WEB_SEARCH_CALL, "index": 0}]},
),
finish_reason="stop",
)
]
)
with patch.object(self.guardrail, "should_run_guardrail", return_value=True):
with pytest.raises(GuardrailRaisedException):
async for _ in self.guardrail.async_post_call_streaming_iterator_hook(
user_api_key_dict=UserAPIKeyAuth(), response=_stream(), request_data={}
):
pass
@pytest.mark.asyncio
async def test_rewrite_mode_removes_the_denied_server_tool_call(self):
rewriting: Final = ToolPermissionGuardrail(
guardrail_name="test-tool-permission",
rules=self.test_rules,
default_action="deny",
on_disallowed_action="rewrite",
)
with patch.object(rewriting, "should_run_guardrail", return_value=True):
result: Final = await rewriting.async_post_call_success_hook(
data={}, user_api_key_dict=UserAPIKeyAuth(), response=self._server_search_response()
)
message: Final = result.choices[0].message
assert message.provider_specific_fields["server_tool_calls"] == []
assert [tool_call.id for tool_call in message.tool_calls] == [_ALLOWED_BASH_CALL["id"]]
assert "Permission denied" in message.content
def test_modify_response_with_permission_errors_keeps_custom_tool_call_in_another_choice(self):
denied_call = ChatCompletionMessageToolCall(
id="call_bash", type="function", function=Function(name="Bash", arguments="{}")
@ -1063,6 +1135,7 @@ class TestToolPermissionGuardrail:
]
assert kept == [[], ["call_patch"]]
class TestToolPermissionGuardrailIntegration:
"""Integration tests for Tool Permission Guardrail"""

31
tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py
View file

@ -50,6 +50,7 @@ from litellm.proxy.mcp_server.guardrail_translation.handler import (
)
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.responses.main import build_web_search_call
from litellm.types.utils import (
CallTypes,
CallTypesLiteral,
@ -1288,6 +1289,36 @@ class TestStreamingTransform:
# end of stream (it can block), matching block_only.
assert guardrail.received_tool_calls
@pytest.mark.asyncio
async def test_claude_server_tool_call_chunks_keep_their_provider_fields(self):
server_fields = {
"server_tool_calls": [
{
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_fetch", "arguments": '{"url": "https://example.com/"}'},
}
],
"web_search_calls": {
"srvtoolu_01": build_web_search_call(
"srvtoolu_01", {"url": "https://example.com/"}, None, tool_name="web_fetch"
).model_dump()
},
}
server_chunk = ModelResponseStream(
choices=[StreamingChoices(index=0, delta=Delta(content=None, provider_specific_fields=server_fields))],
)
out = await _drive_stream(
UnifiedLLMGuardrails(),
_StreamingTextGuardrail(),
[server_chunk, _stream_chunk("done", finish_reason="stop")],
)
delivered = [getattr(item.choices[0].delta, "provider_specific_fields", None) for item in out if item.choices]
assert server_fields in delivered
assert "".join(_delta_text(item) for item in out) == "DONE"
@pytest.mark.asyncio
async def test_per_choice_finish_reason_when_choices_finish_in_different_chunks(self):
"""n>1: a choice finishing before the stream's last chunk keeps its own

44
tests/unit/proxy/guardrails/test_pillar_guardrails.py
View file

@ -34,6 +34,7 @@ from litellm.proxy.guardrails.guardrail_hooks.pillar.pillar import (
build_pillar_response_headers,
)
from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2
from litellm.types.utils import Choices, Message, ModelResponse
# ============================================================================
# FIXTURES
@ -669,6 +670,49 @@ async def test_post_call_hook_with_tool_calls(
assert result == mock_llm_response_with_tools
@pytest.mark.asyncio
async def test_post_call_hook_blocks_a_flagged_claude_server_tool_call(
pillar_guardrail_instance,
sample_request_data,
user_api_key_dict,
pillar_clean_response,
pillar_flagged_response,
):
server_call = {
"id": "srvtoolu_01",
"type": "function",
"function": {"name": "web_fetch", "arguments": '{"url": "https://example.com/"}'},
}
response = ModelResponse(
choices=[
Choices(
message=Message(
role="assistant",
content="Fetched it",
provider_specific_fields={"server_tool_calls": [server_call]},
)
)
]
)
async def _pillar_flags_the_server_call(*_args, **kwargs):
scanned_tool_calls = kwargs["json"]["messages"][-1].get("tool_calls") or []
return pillar_flagged_response if server_call in scanned_tool_calls else pillar_clean_response
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
side_effect=_pillar_flags_the_server_call,
):
with pytest.raises(HTTPException) as excinfo:
await pillar_guardrail_instance.async_post_call_success_hook(
data=sample_request_data,
user_api_key_dict=user_api_key_dict,
response=response,
)
assert excinfo.value.status_code == 400
# =========================================================================
# HEADER CONFIGURATION TESTS
# =========================================================================

224
tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py
View file

@ -33,7 +33,7 @@ from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
from litellm.types.llms.base import HiddenParams
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse
from litellm.types.responses.main import OutputToolSearchCall, build_web_search_call
from litellm.types.utils import (
ChatCompletionMessageToolCall,
@ -4993,6 +4993,159 @@ def test_function_call_tool_id_falls_back_to_unique_id_for_degenerate_call_id():
assert convert(openai)["id"] == "call_tokyo"
_WEATHER_TOOL_USE: Final = {
"type": "tool_use",
"id": "toolu_01Weather",
"name": "get_weather",
"input": {"city": "Paris"},
}
_FETCHED_URL: Final = "https://example.com/"
_BLOCKED_URL: Final = "https://blocked.invalid/"
_ANTHROPIC_WEB_FETCH_TURN: Final[tuple[dict[str, object], ...]] = (
{"type": "server_tool_use", "id": "srvtoolu_01Fetch", "name": "web_fetch", "input": {"url": _FETCHED_URL}},
{
"type": "web_fetch_tool_result",
"tool_use_id": "srvtoolu_01Fetch",
"content": {
"type": "web_fetch_result",
"url": _FETCHED_URL,
"content": {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "Example"}},
},
},
{"type": "server_tool_use", "id": "srvtoolu_02Fetch", "name": "web_fetch", "input": {"url": _BLOCKED_URL}},
{
"type": "web_fetch_tool_result",
"tool_use_id": "srvtoolu_02Fetch",
"content": {"type": "web_fetch_tool_result_error", "error_code": "url_not_allowed"},
},
{"type": "text", "text": "The page is titled Example."},
_WEATHER_TOOL_USE,
)
def _anthropic_turn_on_responses(content: Sequence[Mapping[str, object]]) -> ResponsesAPIResponse:
from unittest.mock import MagicMock
import httpx
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
anthropic_response: Final = {
"id": "msg_01Fetch",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": list(content),
"stop_reason": "tool_use",
"stop_sequence": None,
"usage": {"input_tokens": 100, "output_tokens": 50},
}
raw_response: Final = MagicMock(spec=httpx.Response)
raw_response.json.return_value = anthropic_response
raw_response.status_code = 200
raw_response.headers = {}
chat_response: Final = AnthropicConfig().transform_parsed_response(
completion_response=anthropic_response,
raw_response=raw_response,
model_response=ModelResponse(),
json_mode=False,
prefix_prompt=None,
)
return LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(
request_input="Use the web and check the weather in Paris.",
responses_api_request={
"tools": [
{"type": "web_search_20250305", "name": "web_search"},
{"type": "web_fetch_20250910", "name": "web_fetch"},
{"type": "function", "name": "get_weather", "parameters": {"type": "object"}},
]
},
chat_completion_response=chat_response,
)
def test_anthropic_web_fetch_is_a_hosted_open_page_call_on_responses() -> None:
response: Final = _anthropic_turn_on_responses(_ANTHROPIC_WEB_FETCH_TURN)
assert [
(item.id, item.status, item.action.model_dump()) for item in response.output if item.type == "web_search_call"
] == [
("ws_srvtoolu_01Fetch", "completed", {"type": "open_page", "url": _FETCHED_URL}),
("ws_srvtoolu_02Fetch", "failed", {"type": "open_page", "url": _BLOCKED_URL}),
]
assert [item.name for item in response.output if item.type == "function_call"] == ["get_weather"]
@pytest.mark.parametrize(
("server_tool_use", "expected_action"),
[
(
{
"type": "server_tool_use",
"id": "srvtoolu_01Pending",
"name": "web_fetch",
"input": {"url": _FETCHED_URL},
},
{"type": "open_page", "url": _FETCHED_URL},
),
(
{"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_search", "input": {"query": "python"}},
{"type": "search", "query": "python"},
),
],
)
def test_anthropic_server_web_call_without_a_result_is_an_in_progress_hosted_call_on_responses(
server_tool_use: Mapping[str, object], expected_action: Mapping[str, object]
) -> None:
response: Final = _anthropic_turn_on_responses((server_tool_use, _WEATHER_TOOL_USE))
web_calls: Final = [item for item in response.output if item.type == "web_search_call"]
assert [(item.id, item.status) for item in web_calls] == [("ws_srvtoolu_01Pending", "in_progress")]
assert expected_action.items() <= web_calls[0].action.model_dump().items()
assert [item.name for item in response.output if item.type == "function_call"] == ["get_weather"]
def test_anthropic_code_execution_is_never_a_client_function_call_on_responses() -> None:
response: Final = _anthropic_turn_on_responses(
(
{
"type": "server_tool_use",
"id": "srvtoolu_01Bash",
"name": "bash_code_execution",
"input": {"command": "ls"},
},
{
"type": "bash_code_execution_tool_result",
"tool_use_id": "srvtoolu_01Bash",
"content": {
"type": "bash_code_execution_result",
"stdout": "notes.txt\n",
"stderr": "",
"return_code": 0,
},
},
{
"type": "server_tool_use",
"id": "srvtoolu_02Edit",
"name": "text_editor_code_execution",
"input": {"command": "view", "path": "notes.txt"},
},
{
"type": "server_tool_use",
"id": "srvtoolu_03Bash",
"name": "bash_code_execution",
"input": {"command": "pwd"},
},
_WEATHER_TOOL_USE,
)
)
assert [(item.type, getattr(item, "call_id", item.id)) for item in response.output if item.type != "message"] == [
("code_interpreter_call", "srvtoolu_01Bash"),
("function_call", "toolu_01Weather"),
]
class TestHostedWebSearchReplay:
def test_emitted_hosted_search_output_round_trips_with_client_tool_result(self) -> None:
search_result: Final = {
@ -5063,6 +5216,75 @@ class TestHostedWebSearchReplay:
assert messages[2]["content"] == "Paris is sunny."
assert history == original
@pytest.mark.parametrize(
"leading_text", ((), ({"type": "text", "text": "Let me look that up."},)), ids=("alone", "after_text")
)
@pytest.mark.parametrize(
("server_tool_use", "expected_input"),
(
(
{"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_fetch", "input": {"url": _FETCHED_URL}},
{"url": _FETCHED_URL},
),
(
{"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_search", "input": {"query": "python"}},
{"query": "python"},
),
),
ids=("web_fetch", "web_search"),
)
def test_pending_claude_web_call_replays_to_anthropic_as_its_own_server_tool_use(
self,
leading_text: tuple[Mapping[str, object], ...],
server_tool_use: Mapping[str, object],
expected_input: Mapping[str, object],
) -> None:
response: Final = _anthropic_turn_on_responses((*leading_text, server_tool_use, _WEATHER_TOOL_USE))
history: Final = [
{"role": "user", "content": "Use the web and check the weather in Paris."},
*(item.model_dump(exclude_none=True) for item in response.output),
{"type": "function_call_output", "call_id": "toolu_01Weather", "output": "Paris is sunny."},
]
anthropic_turns: Final = anthropic_messages_pt(
messages=LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=history, responses_api_request={}
),
model="claude-sonnet-5-5",
llm_provider="anthropic",
)
assert [turn["role"] for turn in anthropic_turns] == ["user", "assistant", "user"]
assert [block for block in anthropic_turns[1]["content"] if block["type"] != "text"] == [
{"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": server_tool_use["name"], "input": expected_input},
{"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}},
]
assert [(block["type"], block["tool_use_id"]) for block in anthropic_turns[2]["content"]] == [
("tool_result", "toolu_01Weather")
]
@pytest.mark.parametrize(
("call_id", "status"),
(("ws_srvtoolu_01Fetch", "completed"), ("ws_68a1b2c3d4", "in_progress"), ("srvtoolu_01Fetch", "in_progress")),
ids=("completed-claude-call", "pending-non-claude-call", "pending-unprefixed-id"),
)
def test_web_call_that_is_not_a_pending_claude_call_replays_as_text_only(
self, call_id: str, status: Literal["completed", "in_progress"]
) -> None:
search: Final = ResponseFunctionWebSearch(
id=call_id, type="web_search_call", status=status, action=ActionOpenPage(type="open_page", url=_FETCHED_URL)
)
messages: Final = (
LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message(
input_item=search.model_dump(exclude_none=True)
)
)
assert messages == [
{"role": "assistant", "content": "Hosted web search: " + search.model_dump_json(exclude_none=True)}
]
@pytest.mark.parametrize(
"action",
(

192
tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py
View file

@ -356,6 +356,198 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to
)
_STREAMED_WEB_FETCH_CALL: Final[tuple[dict[str, object], ...]] = (
{
"type": "message_start",
"message": {
"id": "msg_01Fetch",
"type": "message",
"role": "assistant",
"content": [],
"usage": {"input_tokens": 10, "output_tokens": 1},
},
},
{
"type": "content_block_start",
"index": 0,
"content_block": {"type": "server_tool_use", "id": "srvtoolu_01Fetch", "name": "web_fetch"},
},
{
"type": "content_block_delta",
"index": 0,
"delta": {"type": "input_json_delta", "partial_json": '{"url": "https://example.com/"}'},
},
{"type": "content_block_stop", "index": 0},
)
_STREAMED_WEB_FETCH_RESULT: Final[tuple[dict[str, object], ...]] = (
{
"type": "content_block_start",
"index": 1,
"content_block": {
"type": "web_fetch_tool_result",
"tool_use_id": "srvtoolu_01Fetch",
"content": {"type": "web_fetch_result", "url": "https://example.com/"},
},
},
{"type": "content_block_stop", "index": 1},
)
def _streamed_text_then_weather_call(first_index: int) -> tuple[dict[str, object], ...]:
return (
{"type": "content_block_start", "index": first_index, "content_block": {"type": "text", "text": ""}},
{"type": "content_block_delta", "index": first_index, "delta": {"type": "text_delta", "text": "Fetched."}},
{"type": "content_block_stop", "index": first_index},
{
"type": "content_block_start",
"index": first_index + 1,
"content_block": {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {}},
},
{
"type": "content_block_delta",
"index": first_index + 1,
"delta": {"type": "input_json_delta", "partial_json": '{"city": "Paris"}'},
},
{"type": "content_block_stop", "index": first_index + 1},
{
"type": "message_delta",
"delta": {"stop_reason": "tool_use", "stop_sequence": None},
"usage": {"output_tokens": 20},
},
)
def _anthropic_web_fetch_stream_events(fetch_finished: bool) -> tuple[dict[str, object], ...]:
if fetch_finished:
return (*_STREAMED_WEB_FETCH_CALL, *_STREAMED_WEB_FETCH_RESULT, *_streamed_text_then_weather_call(2))
return (*_STREAMED_WEB_FETCH_CALL, *_streamed_text_then_weather_call(1))
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.parametrize(
("fetch_finished", "expected_status", "expected_search_events"),
[
pytest.param(
True,
"completed",
[
ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS,
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING,
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED,
ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
],
id="fetch_finished_in_this_turn",
),
pytest.param(
False,
"in_progress",
[
ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS,
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING,
ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
],
id="fetch_still_waiting_on_its_result",
),
],
)
@pytest.mark.asyncio
async def test_streamed_anthropic_web_fetch_is_a_hosted_open_page_call(
sync_mode: bool,
fetch_finished: bool,
expected_status: str,
expected_search_events: list[ResponsesAPIStreamEvents],
) -> None:
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
parser: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=sync_mode, json_mode=False)
chunks: Final = [parser.chunk_parser(event) for event in _anthropic_web_fetch_stream_events(fetch_finished)]
iterator: Final = LiteLLMCompletionStreamingIterator(
model="claude-sonnet-5-5",
litellm_custom_stream_wrapper=_FakeStreamWrapper(chunks),
request_input="Fetch example.com and check the weather in Paris.",
responses_api_request={
"tools": [
{"type": "web_fetch_20250910", "name": "web_fetch"},
{"type": "function", "name": "get_weather", "parameters": {"type": "object"}},
]
},
custom_llm_provider="anthropic",
)
events: Final = await _collect_events(iterator, sync_mode)
completed: Final = next(
event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
)
assert [
(item.id, item.status, item.action.model_dump())
for item in completed.response.output
if item.type == "web_search_call"
] == [("ws_srvtoolu_01Fetch", expected_status, {"type": "open_page", "url": "https://example.com/"})]
assert [
event.type
for event in events
if getattr(event, "item_id", getattr(getattr(event, "item", None), "id", None)) == "ws_srvtoolu_01Fetch"
] == expected_search_events
assert [item.name for item in completed.response.output if item.type == "function_call"] == ["get_weather"]
assert not any(
event.item.name == "web_fetch"
for event in events
if getattr(event, "type", None) == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED
and event.item.type == "function_call"
)
_STREAMED_PENDING_CODE_EXECUTION: Final[tuple[dict[str, object], ...]] = (
_STREAMED_WEB_FETCH_CALL[0],
{
"type": "content_block_start",
"index": 0,
"content_block": {"type": "server_tool_use", "id": "srvtoolu_01Bash", "name": "bash_code_execution"},
},
{
"type": "content_block_delta",
"index": 0,
"delta": {"type": "input_json_delta", "partial_json": '{"command": "ls"}'},
},
{"type": "content_block_stop", "index": 0},
*_streamed_text_then_weather_call(1),
)
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.asyncio
async def test_streamed_anthropic_code_execution_is_never_a_client_function_call(sync_mode: bool) -> None:
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
parser: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=sync_mode, json_mode=False)
iterator: Final = LiteLLMCompletionStreamingIterator(
model="claude-sonnet-5-5",
litellm_custom_stream_wrapper=_FakeStreamWrapper(
[parser.chunk_parser(event) for event in _STREAMED_PENDING_CODE_EXECUTION]
),
request_input="List the files, then check the weather in Paris.",
responses_api_request={
"tools": [
{"type": "code_execution_20250825", "name": "code_execution"},
{"type": "function", "name": "get_weather", "parameters": {"type": "object"}},
]
},
custom_llm_provider="anthropic",
)
events: Final = await _collect_events(iterator, sync_mode)
item_events: Final = [event for event in events if getattr(event, "item", None) is not None]
assert {event.item.call_id for event in item_events if event.item.type == "function_call"} == {"toolu_01Weather"}
completed: Final = next(
event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
)
assert [item.call_id for item in completed.response.output if item.type == "function_call"] == ["toolu_01Weather"]
def test_tool_calls_present_only_in_final_response_are_emitted_before_completed():
iterator = LiteLLMCompletionStreamingIterator(
model="test-model",

46
ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts
View file

@ -131,6 +131,52 @@ describe("ToolsSection utils", () => {
expect(result[1].callData).toBeUndefined();
});
it("should mark Claude server tools as called from provider_specific_fields.server_tool_calls", () => {
const log: Partial<LogEntry> = {
request_id: "test-server-tools",
proxy_server_request: {
tools: [
{ type: "web_search_20250305", name: "web_search", max_uses: 3 },
{ type: "function", function: { name: "get_weather", description: "Get weather" } },
{ type: "function", function: { name: "send_email", description: "Send email" } },
],
},
response: {
choices: [
{
message: {
tool_calls: [
{
id: "toolu_01Weather",
type: "function",
function: { name: "get_weather", arguments: '{"location": "Paris"}' },
},
],
provider_specific_fields: {
server_tool_calls: [
{
id: "srvtoolu_01Search",
type: "function",
function: { name: "web_search", arguments: '{"query": "paris weather"}' },
},
],
},
},
},
],
},
};
const result = parseToolsFromLog(log as LogEntry);
expect(result.map((tool) => [tool.name, tool.called, tool.callData?.id])).toEqual([
["web_search", true, "srvtoolu_01Search"],
["get_weather", true, "toolu_01Weather"],
["send_email", false, undefined],
]);
expect(result[0].callData?.arguments).toEqual({ query: "paris weather" });
});
it("should handle string format request and response", () => {
const log: Partial<LogEntry> = {
request_id: "test-5",

25
ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts
View file

@ -42,6 +42,24 @@ function extractToolsFromRequest(log: LogEntry): ToolDefinition[] {
return [];
}
interface ChatMessageToolCallFields {
tool_calls?: unknown;
provider_specific_fields?: { server_tool_calls?: unknown } | null;
}
function toolCallList(value: unknown): ToolCall[] {
return Array.isArray(value) ? value : [];
}
function extractChatMessageToolCalls(message: ChatMessageToolCallFields | undefined): ToolCall[] | null {
const serverToolCalls = message?.provider_specific_fields?.server_tool_calls;
const clientToolCalls = message?.tool_calls;
if (!Array.isArray(serverToolCalls) && !Array.isArray(clientToolCalls)) {
return null;
}
return [...toolCallList(serverToolCalls), ...toolCallList(clientToolCalls)];
}
/**
* Extract tool calls from response data
*/
@ -53,10 +71,9 @@ function extractToolCallsFromResponse(log: LogEntry): ToolCall[] {
// OpenAI format: response.choices[0].message.tool_calls
const choices = responseData.choices;
if (Array.isArray(choices) && choices.length > 0) {
const firstChoice = choices[0];
const message = firstChoice.message;
if (message && Array.isArray(message.tool_calls)) {
return message.tool_calls;
const chatToolCalls = extractChatMessageToolCalls(choices[0].message);
if (chatToolCalls) {
return chatToolCalls;
}
}