This commit is contained in:
JeongCheolLee 2026-10-29 18:38:59 +00:00 • committed by GitHub
commit f46c7645f0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 1672 additions and 147 deletions

View file

@ -44,6 +44,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
@ -315,7 +316,10 @@ def _to_dd_tool_calls(message: Mapping[str, object]) -> tuple[ToolCall, ...]:
tool_id=_text_field(tool_call, "id"),
type=_text_field(tool_call, "type", "function"),
)
for tool_call in (raw_tool_calls if isinstance(raw_tool_calls, list) else ())
for tool_call in (
*anthropic_server_tool_call_params(message),
*(raw_tool_calls if isinstance(raw_tool_calls, list) else ()),
)
if isinstance(tool_call, dict)
for function in [_mapping_field(tool_call, "function")]
)

View file

@ -12,6 +12,8 @@ from typing import Final
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue
from litellm.integrations.otel.model.payloads import LLMCallSpanData, ToolDefinition
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
@dataclass(frozen=True, slots=True)
@ -141,12 +143,13 @@ def output_messages(data: LLMCallSpanData) -> list:
def message_tool_calls(message: object) -> tuple[MessageToolCall, ...]:
tool_call_values: Final = (
*anthropic_server_tool_call_params(message),
*parts_of(message_field(message, "tool_calls")),
)
if not isinstance(message, dict):
return ()
tool_calls: Final = message.get("tool_calls")
if not isinstance(tool_calls, (list, tuple)) or not tool_calls:
return ()
return tuple(tool_call for value in tool_calls if (tool_call := _message_tool_call(value)) is not None)
return tuple(tool_call for value in tool_call_values if (tool_call := _message_tool_call(value)) is not None)
def _message_tool_call(value: object) -> MessageToolCall | None:

View file

@ -2004,12 +2004,39 @@ def _tool_call_ids_with_a_kept_server_result(message: object) -> frozenset[str]:
return frozenset()
return frozenset(
tool_call_id
for tool_call in parts_of(message_field(message, "tool_calls"))
for tool_call in (
*parts_of(message_field(message, "tool_calls")),
*parts_of(message_field(provider_specific_fields, "server_tool_calls")),
)
if isinstance(tool_call_id := message_field(tool_call, "id"), str)
and tool_call_is_rebuilt_as_server_tool_use(tool_call_id, provider_specific_fields)
)
def _assistant_tool_calls_with_server_tool_calls(
message: ChatCompletionAssistantMessage,
) -> Sequence[ChatCompletionAssistantToolCall] | None:
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
tool_calls: Final = message.get("tool_calls")
server_tool_calls: Final = anthropic_server_tool_call_params(message)
if not server_tool_calls:
return tool_calls
client_tool_call_ids: Final = frozenset(tool_call["id"] for tool_call in tool_calls or ())
return (
*(tool_call for tool_call in server_tool_calls if tool_call["id"] not in client_tool_call_ids),
*(tool_calls or ()),
)
def _server_tool_call_ids(message: object) -> frozenset[str]:
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
return frozenset(
tool_call["id"] for tool_call in anthropic_server_tool_call_params(message) if tool_call["id"] is not None
)
def _server_tool_answers_replaced_after(messages: Sequence[object], index: int) -> frozenset[str]:
kept_server_result_ids: Final = _tool_call_ids_with_a_kept_server_result(messages[index])
if not kept_server_result_ids:
@ -2069,10 +2096,11 @@ _AnthropicToolInvokeItem: TypeAlias = AnthropicMessagesToolUseParam | dict[str,
def convert_to_anthropic_tool_invoke(
tool_calls: list[ChatCompletionAssistantToolCall],
tool_calls: Sequence[ChatCompletionAssistantToolCall],
web_search_results: Sequence[object] | None = None,
tool_results: Sequence[object] | None = None,
answered_tool_ids: Container[str] = frozenset(),
server_tool_call_ids: Container[str] = frozenset(),
) -> list[_AnthropicToolInvokeItem]:
"""
OpenAI tool invokes:
@ -2150,6 +2178,10 @@ def convert_to_anthropic_tool_invoke(
anthropic_tool_invoke.append(server_tool_result)
elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids:
continue
elif tool_id in server_tool_call_ids and tool_id not in answered_tool_ids:
anthropic_tool_invoke.append(
{"type": "server_tool_use", "id": tool_id, "name": tool_name, "input": tool_input}
)
else:
sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id)
_anthropic_tool_use_param = AnthropicMessagesToolUseParam(
@ -3109,7 +3141,7 @@ def anthropic_messages_pt(
# If so, we need to interleave thinking blocks with tool call groups
# to preserve the original content block ordering.
# Fixes: https://github.com/BerriAI/litellm/issues/23047
assistant_tool_calls = assistant_content_block.get("tool_calls")
assistant_tool_calls = _assistant_tool_calls_with_server_tool_calls(assistant_content_block)
_has_server_tool_calls = False
if assistant_tool_calls is not None:
for _tc in assistant_tool_calls:
@ -3142,6 +3174,7 @@ def anthropic_messages_pt(
web_search_results=_web_search_results_tc,
tool_results=_tool_results_tc,
answered_tool_ids=client_answered_tool_ids,
server_tool_call_ids=_server_tool_call_ids(assistant_content_block),
)
# Group tool invoke results into (server_tool_use, result) pairs
@ -3307,6 +3340,7 @@ def anthropic_messages_pt(
web_search_results=_web_search_results,
tool_results=_tool_results,
answered_tool_ids=client_answered_tool_ids,
server_tool_call_ids=_server_tool_call_ids(assistant_content_block),
)
# Prevent "tool_use ids must be unique" errors by filtering duplicates

View file

@ -26,6 +26,7 @@ from typing_extensions import ReadOnly, TypedDict, assert_never
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.common_utils import anthropic_server_tool_calls
from litellm.llms.anthropic.messages.adapters.transformation import (
OPENAI_MAX_TOOL_NAME_LENGTH,
LiteLLMAnthropicMessagesAdapter,
@ -1317,9 +1318,9 @@ class AnthropicMessagesHandler(BaseTranslation):
if built_response is not None and hasattr(built_response, "choices") and built_response.choices:
model_response: Final = cast(ModelResponse, built_response)
first_choice: Final = cast(Choices, model_response.choices[0])
tool_calls_list: Final = cast(
list[ChatCompletionMessageToolCall] | None,
first_choice.message.tool_calls,
tool_calls_list: Final = (
*anthropic_server_tool_calls(first_choice.message),
*cast(list[ChatCompletionMessageToolCall], first_choice.message.tool_calls or []),
)
string_so_far = first_choice.message.content
pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ())
@ -1327,7 +1328,7 @@ class AnthropicMessagesHandler(BaseTranslation):
if string_so_far:
guardrail_inputs["texts"] = [string_so_far]
if tool_calls_list:
guardrail_inputs["tool_calls"] = tool_calls_list
guardrail_inputs["tool_calls"] = list(tool_calls_list)
try:
prepared_request_data = self._prepare_request_data(

View file

@ -54,6 +54,7 @@ from litellm.types.llms.openai import (
ChatCompletionToolCallFunctionChunk,
)
from litellm.types.responses.main import (
HOSTED_WEB_TOOL_NAMES,
OutputCodeInterpreterCall,
build_code_interpreter_log_outputs,
build_web_search_call,
@ -754,6 +755,8 @@ class ModelResponseIterator:
self._server_tool_inputs: dict[str, object] = {}
self.tool_results: list[dict[str, Any]] = []
self._current_server_tool_id: str | None = None
self._current_server_tool_name: str = ""
self._server_tool_calls: tuple[ChatCompletionToolCallChunk, ...] = ()
self._container_id: str | None = None
@property
@ -824,10 +827,9 @@ class ModelResponseIterator:
return text, tool_use, thinking_blocks, provider_specific_fields, reasoning_content
self.content_blocks.append(content_block)
if "partial_json" in content_block["delta"]:
# Only emit tool calls if we're in a tool_use or server_tool_use block
# web_search_tool_result blocks also have input_json_delta but should not be treated as tool calls
# See: https://github.com/BerriAI/litellm/issues/17254
if self.current_content_block_type in ("tool_use", "server_tool_use"):
if self.current_content_block_type == "tool_use":
tool_use = cast(
ChatCompletionToolCallChunk,
{
@ -926,7 +928,7 @@ class ModelResponseIterator:
def _web_search_call_snapshot(self) -> dict[str, object]:
return dict(self._web_search_calls)
def _complete_web_search_call(self, result: dict[str, object]) -> None:
def _complete_web_search_call(self, result: dict[str, object], tool_name: str) -> None:
tool_use_id: Final = result.get("tool_use_id")
if not isinstance(tool_use_id, str) or tool_use_id not in self._web_search_calls:
return
@ -934,6 +936,7 @@ class ModelResponseIterator:
tool_id=tool_use_id,
tool_input=self._server_tool_inputs.get(tool_use_id, {}),
result=result,
tool_name=tool_name,
)
def _build_code_interpreter_results(self) -> list:
@ -1013,7 +1016,6 @@ class ModelResponseIterator:
content_block_start["content_block"]["type"] == "tool_use"
or content_block_start["content_block"]["type"] == "server_tool_use"
):
self.tool_index += 1
# Reverse-map the (sanitized) tool name back to the
# caller's original. No-op when the map is empty.
_stream_tool_name = content_block_start["content_block"]["name"]
@ -1022,35 +1024,39 @@ class ModelResponseIterator:
# Use empty string for arguments in content_block_start - actual arguments
# come in subsequent content_block_delta chunks and get accumulated.
# Using str(input) here would prepend '{}' causing invalid JSON accumulation.
tool_use = ChatCompletionToolCallChunk(
id=content_block_start["content_block"]["id"],
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=_stream_tool_name,
arguments="",
),
index=self.tool_index,
)
if content_block_start["content_block"]["type"] == "tool_use":
self.tool_index += 1
tool_use = ChatCompletionToolCallChunk(
id=content_block_start["content_block"]["id"],
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=_stream_tool_name,
arguments="",
),
index=self.tool_index,
)
# Track server tool use inputs for code_interpreter_results.
# The initial input in content_block_start is typically {}
# for streaming; the full input arrives via input_json_delta
# and is assembled at content_block_stop.
if content_block_start["content_block"]["type"] == "server_tool_use":
self._current_server_tool_id = content_block_start["content_block"]["id"]
self._current_server_tool_name = _stream_tool_name
tool_input: Final = content_block_start["content_block"].get("input", {})
self._server_tool_inputs[self._current_server_tool_id] = tool_input
if _stream_tool_name == "web_search":
if _stream_tool_name in HOSTED_WEB_TOOL_NAMES:
self._web_search_calls[self._current_server_tool_id] = build_web_search_call(
self._current_server_tool_id,
tool_input,
{"content": []},
status="in_progress",
tool_name=_stream_tool_name,
)
provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
# Include caller information if present (for programmatic tool calling)
if "caller" in content_block_start["content_block"]:
caller_data: Final = content_block_start["content_block"]["caller"]
if caller_data:
if caller_data and tool_use is not None:
tool_use["caller"] = cast(dict[str, Any], caller_data)
elif content_block_start["content_block"]["type"] == "redacted_thinking":
(
@ -1081,7 +1087,7 @@ class ModelResponseIterator:
# The full content comes in content_block_start, not in deltas
# See: https://github.com/BerriAI/litellm/issues/17737
self.web_search_results.append(content_block_start["content_block"])
self._complete_web_search_call(content_block_start["content_block"])
self._complete_web_search_call(content_block_start["content_block"], "web_search")
provider_specific_fields["web_search_results"] = self.web_search_results
provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
elif content_type == "web_fetch_tool_result":
@ -1089,7 +1095,9 @@ class ModelResponseIterator:
# The full content comes in content_block_start, not in deltas
# Fixes: https://github.com/BerriAI/litellm/issues/18137
self.web_search_results.append(content_block_start["content_block"])
self._complete_web_search_call(content_block_start["content_block"], "web_fetch")
provider_specific_fields["web_search_results"] = self.web_search_results
provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
else:
self.tool_results.append(content_block_start["content_block"])
provider_specific_fields["tool_results"] = self.tool_results
@ -1104,7 +1112,7 @@ class ModelResponseIterator:
# check if tool call content block - only for tool_use and server_tool_use blocks
if self.current_content_block_type in ("tool_use", "server_tool_use"):
is_empty: Final = self.check_empty_tool_call_args()
if is_empty:
if is_empty and self.current_content_block_type == "tool_use":
tool_use = ChatCompletionToolCallChunk(
id=None,
type="function",
@ -1128,6 +1136,29 @@ class ModelResponseIterator:
self._server_tool_inputs[self._current_server_tool_id] = json.loads(args)
except (json.JSONDecodeError, TypeError):
pass
server_tool_input: Final = self._server_tool_inputs.get(self._current_server_tool_id, {})
if self._current_server_tool_id in self._web_search_calls:
self._web_search_calls[self._current_server_tool_id] = build_web_search_call(
self._current_server_tool_id,
server_tool_input,
{"content": []},
status="in_progress",
tool_name=self._current_server_tool_name,
)
provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
self._server_tool_calls = (
*self._server_tool_calls,
ChatCompletionToolCallChunk(
id=self._current_server_tool_id,
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=self._current_server_tool_name,
arguments=json.dumps(server_tool_input),
),
index=len(self._server_tool_calls),
),
)
provider_specific_fields["server_tool_calls"] = list(self._server_tool_calls)
self._current_server_tool_id = None
if self.is_response_format_tool:
text = self._flush_response_format_buffer()

View file

@ -7,6 +7,7 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, cast
import httpx
from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
@ -62,6 +63,7 @@ from litellm.types.llms.anthropic import (
ANTHROPIC_ADVISOR_TOOL_TYPE,
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX,
ANTHROPIC_TOOL_SEARCH_TOOL_TYPES,
AllAnthropicPassThroughMessageValues,
AllAnthropicToolsValues,
@ -94,6 +96,7 @@ from litellm.types.llms.openai import (
OpenAIWebSearchOptions,
)
from litellm.types.responses.main import (
HOSTED_WEB_TOOL_NAMES,
OutputCodeInterpreterCall,
build_code_interpreter_log_outputs,
build_web_search_call,
@ -251,6 +254,29 @@ def _validated_defer_loading(value: object) -> bool:
raise ValueError("defer_loading must be a boolean") from exc
def _hosted_web_tool_use(block: object) -> tuple[str, str, object] | None:
match block:
case {
"type": "server_tool_use",
"name": str() as name,
"id": str() as call_id,
"input": object() as tool_input,
} if name in HOSTED_WEB_TOOL_NAMES:
return call_id, name, tool_input
case _:
return None
def _hosted_web_call(call_id: str, tool_name: str, tool_input: object, result: object) -> ResponseFunctionWebSearch:
return build_web_search_call(
tool_id=call_id,
tool_input=tool_input,
result=result,
status=None if result is not None else "in_progress",
tool_name=tool_name,
)
class _AnthropicUsageIteration(TypedDict, total=False):
"""One entry of the ``usage.iterations`` array on an Anthropic response."""
@ -2850,6 +2876,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
return usage
@staticmethod
def _is_server_tool_call(tool_call: ChatCompletionToolCallChunk) -> bool:
tool_call_id: Final = tool_call["id"]
return tool_call_id is not None and tool_call_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
def _build_code_by_id_map(self, tool_calls: list[ChatCompletionToolCallChunk]) -> dict[str, str]:
code_by_id: Final[dict[str, str]] = {}
for tc in tool_calls:
@ -2894,29 +2925,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
self,
web_search_results: Sequence[object],
completion_response: Mapping[str, object],
) -> list[object]:
) -> list[ResponseFunctionWebSearch]:
content: Final = completion_response.get("content")
blocks: Final = content if isinstance(content, Sequence) else ()
inputs: Final = {
call_id: tool_input
for block in blocks
if isinstance(block, Mapping)
and block.get("type") == "server_tool_use"
and block.get("name") == "web_search"
and isinstance((call_id := block.get("id")), str)
and isinstance((tool_input := block.get("input")), Mapping)
results_by_call: Final = {
(tool_use_id, result.get("type")): result
for result in web_search_results
if isinstance(result, dict) and isinstance((tool_use_id := result.get("tool_use_id")), str)
}
return [
build_web_search_call(
tool_id=tool_use_id,
tool_input=inputs.get(tool_use_id, {}),
result=result,
)
for result in web_search_results
if isinstance(result, dict)
and result.get("type") == "web_search_tool_result"
and isinstance((tool_use_id := result.get("tool_use_id")), str)
and tool_use_id in inputs
_hosted_web_call(call_id, tool_name, tool_input, results_by_call.get((call_id, f"{tool_name}_tool_result")))
for call_id, tool_name, tool_input in filter(None, map(_hosted_web_tool_use, blocks))
]
def _build_provider_specific_fields(
@ -2940,10 +2959,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if web_search_results is not None:
provider_specific_fields["web_search_results"] = web_search_results
provider_specific_fields["web_search_calls"] = self._build_web_search_calls(
web_search_results,
completion_response,
)
web_search_calls: Final = self._build_web_search_calls(web_search_results or (), completion_response)
if web_search_calls:
provider_specific_fields["web_search_calls"] = web_search_calls
if tool_results is not None:
provider_specific_fields["tool_results"] = tool_results
@ -2965,6 +2984,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if compaction_blocks is not None:
provider_specific_fields["compaction_blocks"] = compaction_blocks
server_tool_calls: Final = [tc for tc in tool_calls if self._is_server_tool_call(tc)]
if server_tool_calls:
provider_specific_fields["server_tool_calls"] = server_tool_calls
refusal_fields: Final = refusal_provider_specific_fields(
REFUSAL_STOP_FIELDS.validate_python(completion_response)
)
@ -3024,6 +3047,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
):
text_content = prefix_prompt + text_content
client_tool_calls: Final = [tc for tc in tool_calls if not self._is_server_tool_call(tc)]
provider_specific_fields: Final = self._build_provider_specific_fields(
completion_response,
citations,
@ -3036,7 +3060,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
json_mode_message, tool_calls_for_message, json_extra_content = self._resolve_json_mode_non_streaming(
json_mode=json_mode,
tool_calls=tool_calls,
tool_calls=client_tool_calls,
response_format_schema=response_format_schema,
)
merged_text = text_content or ""

View file

@ -60,10 +60,10 @@ from litellm.types.llms.anthropic import (
AnthropicThinkingParam,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import AllMessageValues
from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantToolCall
from litellm.types.proxy.auth.special_headers import SpecialHeaders
from litellm.types.proxy.model_listing import ModelInfoResponse
from litellm.types.utils import LlmProviders
from litellm.types.utils import ChatCompletionMessageToolCall, LlmProviders
_MessageT = TypeVar("_MessageT")
@ -116,6 +116,7 @@ _DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)")
_CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
_SERVER_TOOL_CALLS_ADAPTER: Final = TypeAdapter(tuple[ChatCompletionAssistantToolCall, ...])
_DECODED_JSON: Final = TypeAdapter(object)
@ -2074,6 +2075,21 @@ def _replayed_server_tool_use(block: object) -> _ReplayedServerToolUse | None:
return None
def anthropic_server_tool_call_params(message: object) -> tuple[ChatCompletionAssistantToolCall, ...]:
fields: Final = validated_object_mapping(message_field(message, "provider_specific_fields"))
try:
return _SERVER_TOOL_CALLS_ADAPTER.validate_python(fields.get("server_tool_calls", ()) if fields else ())
except ValidationError:
return ()
def anthropic_server_tool_calls(message: object) -> tuple[ChatCompletionMessageToolCall, ...]:
return tuple(
ChatCompletionMessageToolCall(id=call["id"], type=call["type"], function=dict(call["function"]))
for call in anthropic_server_tool_call_params(message)
)
def tool_call_is_rebuilt_as_server_tool_use(
tool_call_id: object, provider_specific_fields: object, answered_tool_ids: Container[str] = frozenset()
) -> bool:

View file

@ -45,6 +45,25 @@ def _thought_signature(provider_specific_fields: object) -> str | None:
return signature if isinstance(signature, str) else None
def _object_sequence(value: object) -> Sequence[object] | None:
return value if isinstance(value, (list, tuple)) else None
def _server_tool_blocks(message: object) -> tuple[Mapping[str, object], ...]:
server_tool_calls: Final = anthropic_server_tool_call_params(message)
if not server_tool_calls:
return ()
fields: Final = _as_string_mapping(_optional_attr(message, "provider_specific_fields")) or {}
return tuple(
convert_to_anthropic_tool_invoke(
server_tool_calls,
web_search_results=_object_sequence(fields.get("web_search_results")),
tool_results=_object_sequence(fields.get("tool_results")),
server_tool_call_ids=frozenset(tool_call["id"] for tool_call in server_tool_calls),
)
)
def _compaction_blocks(provider_specific_fields: object) -> tuple[Mapping[str, object], ...]:
fields: Final = _as_string_mapping(provider_specific_fields)
raw_blocks: Final = fields.get("compaction_blocks") if fields is not None else None
@ -130,11 +149,13 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
)
from litellm.litellm_core_utils.prompt_templates.factory import (
THOUGHT_SIGNATURE_SEPARATOR,
convert_to_anthropic_tool_invoke,
)
from litellm.litellm_core_utils.reasoning_effort_utils import (
reasoning_effort_from_thinking,
)
from litellm.llms.anthropic.common_utils import (
anthropic_server_tool_call_params,
eager_input_streaming_flag,
is_empty_unsigned_thinking_block,
normalize_anthropic_tool_use_id,
@ -1428,6 +1449,7 @@ class LiteLLMAnthropicMessagesAdapter:
).model_dump()
)
new_content.extend(dict(block) for block in _server_tool_blocks(choice.message))
# Handle text content
if choice.message.content is not None and (choice.message.content != "" or not compaction_blocks):
new_content.append(

View file

@ -21,6 +21,7 @@ from collections.abc import Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Union, cast
from pydantic import TypeAdapter, ValidationError
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
@ -29,6 +30,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i
replayed_tool_search_tool_names,
)
from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
StreamingScanKey,
@ -50,7 +52,12 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
stream_item_items,
unappliable_request_rewrite,
)
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantToolCall,
ChatCompletionToolCallFunctionChunk,
ChatCompletionToolParam,
)
from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
coerce_stream_holdback_value,
)
@ -436,9 +443,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
text_task_mappings=text_task_mappings,
tool_call_task_mappings=tool_call_task_mappings,
)
server_tool_call_tasks: Final = tuple(_server_tool_call_tasks(response.choices))
tool_calls_with_server_calls: Final = [
*tool_calls_to_check,
*(_server_tool_call_input(call) for _, call in server_tool_call_tasks),
]
# Step 2: Apply guardrail to all texts and tool calls in batch
if texts_to_check or tool_calls_to_check:
if texts_to_check or tool_calls_with_server_calls:
# Use the real request_data if provided (proxy path), otherwise
# create a standalone dict (SDK / direct-call path).
if request_data is None:
@ -452,8 +464,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
if images_to_check:
inputs["images"] = images_to_check
if tool_calls_to_check:
inputs["tool_calls"] = tool_calls_to_check
if tool_calls_with_server_calls:
inputs["tool_calls"] = tool_calls_with_server_calls
# Include model information from the response if available
if hasattr(response, "model") and response.model:
inputs["model"] = response.model
@ -469,8 +481,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
returned_tool_calls: Final = guardrailed_inputs.get("tool_calls")
guardrailed_tool_calls: Final[list[dict[str, object]]] = (
cast(list[dict[str, object]], returned_tool_calls)
if isinstance(returned_tool_calls, list) and len(returned_tool_calls) == len(tool_calls_to_check)
else tool_calls_to_check
if isinstance(returned_tool_calls, list)
and len(returned_tool_calls) == len(tool_calls_with_server_calls)
else tool_calls_with_server_calls
)
# Step 3: Map guardrail responses back to original response structure
@ -488,6 +501,11 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
tool_calls=guardrailed_tool_calls,
task_mappings=tool_call_task_mappings,
)
self._apply_guardrail_responses_to_output_server_tool_calls(
response=response,
guardrailed_tool_calls=guardrailed_tool_calls[len(tool_calls_to_check) :],
task_mappings=tuple(mapping for mapping, _ in server_tool_call_tasks),
)
verbose_proxy_logger.debug("OpenAI Chat Completions: Processed output response: %s", response)
@ -681,6 +699,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
pre_guardrail_tool_calls=pre_guardrail_tool_calls,
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
)
self._write_ended_stream_server_tool_call_rewrites(
responses_so_far=responses_so_far,
guardrailed_response=model_response,
)
@staticmethod
def _rebuild_ended_stream_per_choice(
@ -939,6 +961,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if choice.message.tool_calls and isinstance(choice.message.tool_calls, list):
if len(choice.message.tool_calls) > 0:
return True
if anthropic_server_tool_call_params(choice.message):
return True
elif isinstance(response, ModelResponseStream):
for streaming_choice in response.choices:
if isinstance(streaming_choice, litellm.StreamingChoices):
@ -949,6 +973,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if streaming_choice.delta.tool_calls and isinstance(streaming_choice.delta.tool_calls, list):
if len(streaming_choice.delta.tool_calls) > 0:
return True
if anthropic_server_tool_call_params(streaming_choice.delta):
return True
return False
def _extract_output_text_images_and_tool_calls(
@ -1099,6 +1125,25 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if "name" in func_dict:
existing_tool_call.function.name = func_dict["name"]
@staticmethod
def _apply_guardrail_responses_to_output_server_tool_calls(
response: "ModelResponse",
guardrailed_tool_calls: Sequence[object],
task_mappings: Sequence[tuple[int, int]],
) -> None:
guardrailed_by_task: Final = MappingProxyType(dict(zip(task_mappings, guardrailed_tool_calls)))
for choice_idx, choice in enumerate(response.choices):
server_tool_calls = anthropic_server_tool_call_params(choice.message)
rewritten = tuple(
_rewritten_server_tool_call(server_tool_call, guardrailed_by_task.get((choice_idx, server_idx)))
for server_idx, server_tool_call in enumerate(server_tool_calls)
)
if rewritten != server_tool_calls:
choice.message.provider_specific_fields = {
**(choice.message.provider_specific_fields or {}),
"server_tool_calls": list(rewritten),
}
@staticmethod
def _string_choice_contents(response: "ModelResponse") -> tuple[str | None, ...]:
return tuple(
@ -1211,6 +1256,32 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
for fragment in tail:
fragment.function.arguments = ""
@staticmethod
def _write_ended_stream_server_tool_call_rewrites(
responses_so_far: Sequence["ModelResponseStream"],
guardrailed_response: "ModelResponse",
) -> None:
guardrailed_by_id: Final = MappingProxyType(
{
(guardrailed_response.choices[choice_position].index, server_tool_call["id"]): server_tool_call
for (choice_position, _), server_tool_call in _server_tool_call_tasks(guardrailed_response.choices)
}
)
if not guardrailed_by_id:
return
for response in responses_so_far:
for choice in response.choices:
streamed = anthropic_server_tool_call_params(choice.delta)
rewritten = tuple(
guardrailed_by_id.get((choice.index, server_tool_call["id"]), server_tool_call)
for server_tool_call in streamed
)
if rewritten != streamed:
choice.delta.provider_specific_fields = {
**(choice.delta.provider_specific_fields or {}),
"server_tool_calls": list(rewritten),
}
async def _apply_guardrail_responses_to_output_streaming(
self,
responses: list["ModelResponseStream"],
@ -1400,7 +1471,42 @@ def _stream_chunk_choices(item: object) -> Sequence[object]:
def _streamed_delta_tool_calls(delta: object) -> tuple[object, ...]:
function_call: Final = stream_item_field(delta, "function_call")
legacy: Final = () if function_call is None else (function_call,)
return stream_item_items(delta, "tool_calls") + legacy
return anthropic_server_tool_call_params(delta) + stream_item_items(delta, "tool_calls") + legacy
def _server_tool_call_tasks(
choices: Sequence[object],
) -> Iterator[tuple[tuple[int, int], ChatCompletionAssistantToolCall]]:
for choice_idx, choice in enumerate(choices):
if not isinstance(choice, Choices):
continue
for server_idx, server_tool_call in enumerate(anthropic_server_tool_call_params(choice.message)):
yield (choice_idx, server_idx), server_tool_call
def _server_tool_call_input(server_tool_call: ChatCompletionAssistantToolCall) -> dict[str, object]:
return {
"id": server_tool_call["id"],
"type": server_tool_call["type"],
"function": dict(server_tool_call["function"]),
}
_TOOL_CALL_FUNCTION_ADAPTER: Final = TypeAdapter(ChatCompletionToolCallFunctionChunk)
def _rewritten_server_tool_call(
server_tool_call: ChatCompletionAssistantToolCall, guardrailed: object
) -> ChatCompletionAssistantToolCall:
try:
function: Final = _TOOL_CALL_FUNCTION_ADAPTER.validate_python(stream_item_field(guardrailed, "function"))
except ValidationError:
return server_tool_call
return ChatCompletionAssistantToolCall(
id=server_tool_call["id"],
type=server_tool_call["type"],
function={**server_tool_call["function"], **function},
)
def _released_choices(responses_so_far: Sequence[object]) -> Iterator[object]:

View file

@ -63,11 +63,26 @@ def response_tool_call_names(completion_response: object) -> tuple[str, ...]:
)
return tuple(
stripped
for name in chain(called_names, _auto_executed_mcp_tool_names(completion_response))
for name in chain(
_server_tool_call_names(completion_response),
called_names,
_auto_executed_mcp_tool_names(completion_response),
)
if isinstance(name, str) and (stripped := name.strip())
)
def _server_tool_call_names(completion_response: object) -> tuple[str | None, ...]:
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
server_tool_calls: Final = chain.from_iterable(
anthropic_server_tool_call_params(message_field(choice, "message"))
for choice in parts_of(message_field(completion_response, "choices"))
)
return tuple(server_tool_call["function"].get("name") for server_tool_call in server_tool_calls)
class _ExecutedTool(BaseModel):
model_config = ConfigDict(frozen=True)

View file

@ -19,6 +19,7 @@ from litellm.integrations.custom_guardrail import (
CustomGuardrail,
log_guardrail_information,
)
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params, anthropic_server_tool_calls
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.callback_utils import (
add_guardrail_to_applied_guardrails_header,
@ -515,7 +516,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
for choice_index, choice in enumerate(response.choices):
if isinstance(choice, Choices):
for tool in choice.message.tool_calls or []:
for tool in (*(choice.message.tool_calls or ()), *anthropic_server_tool_calls(choice.message)):
tool_calls.append(tool)
legacy_tool_call = self._legacy_function_call_to_tool_call(
getattr(choice.message, "function_call", None), choice_index
@ -784,6 +785,23 @@ class ToolPermissionGuardrail(CustomGuardrail):
choice.message.tool_calls = filtered_tool_calls if filtered_tool_calls else None
if (server_tool_calls := anthropic_server_tool_call_params(choice.message)) and any(
server_tool_call["id"] in error_results for server_tool_call in server_tool_calls
):
error_messages.extend(
error_results[server_tool_call["id"]].content
for server_tool_call in server_tool_calls
if server_tool_call["id"] in error_results
)
choice.message.provider_specific_fields = {
**(choice.message.provider_specific_fields or {}),
"server_tool_calls": [
server_tool_call
for server_tool_call in server_tool_calls
if server_tool_call["id"] not in error_results
],
}
legacy_tool_call = self._legacy_function_call_to_tool_call(
getattr(choice.message, "function_call", None), choice_index
)

View file

@ -9,6 +9,7 @@ import litellm
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items
from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend
from litellm.llms.anthropic.common_utils import anthropic_server_tool_calls
from litellm.main import stream_chunk_builder
from litellm.responses.litellm_completion_transformation.compaction_items import compaction_output_item
from litellm.responses.litellm_completion_transformation.custom_tools import (
@ -486,13 +487,16 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
try:
message: Final = litellm_complete_object.choices[0].message
tool_calls = getattr(message, "tool_calls", None)
server_tool_calls: Final = anthropic_server_tool_calls(message)
tool_calls: Final = [*server_tool_calls, *(getattr(message, "tool_calls", None) or ())]
except Exception:
tool_calls = None
if not tool_calls or not isinstance(tool_calls, list):
return
if not tool_calls:
return
server_tool_call_ids: Final = frozenset(tool_call.id for tool_call in server_tool_calls)
for tc in tool_calls:
call_id_raw = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)
if not call_id_raw:
@ -518,6 +522,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
if (search_execution := self._tool_search_execution(call_id, fn_name)) is not None:
self._queue_tool_search_call_done(call_id, output_index, fn_args, search_execution)
continue
if call_id in server_tool_call_ids:
continue
# Track if this is a new tool call that wasn't streamed
is_new_tool_call = call_id not in self._tool_item_id_by_call_id
@ -592,10 +598,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
}
),
)
finished_events: Final = (
()
if item.status == "in_progress"
else ((ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),)
)
for event_type, event_class in (
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, WebSearchCallInProgressEvent),
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, WebSearchCallSearchingEvent),
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),
*finished_events,
):
self._sequence_number += 1
event = event_class(type=event_type, output_index=output_index, item_id=item.id)

View file

@ -31,6 +31,7 @@ from openai.types.chat.chat_completion_named_tool_choice_param import (
from openai.types.responses import ResponseFunctionToolCall, ResponseFunctionWebSearch
from openai.types.responses.response_compaction_item import ResponseCompactionItem
from openai.types.responses.response_create_params import ResponseInputParam
from openai.types.responses.response_function_web_search import ActionOpenPage, ActionSearch
from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam
from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam
from openai.types.responses.tool_param import FunctionToolParam
@ -56,7 +57,12 @@ from litellm.litellm_core_utils.hidden_params import (
set_hidden_params,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control
from litellm.llms.anthropic.common_utils import (
anthropic_server_tool_call_params,
anthropic_server_tool_calls,
supports_anthropic_cache_control,
validated_object_mapping,
)
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support
from litellm.llms.openrouter.chat.transformation import openrouter_model_supports_cache_control
@ -66,11 +72,12 @@ from litellm.responses.litellm_completion_transformation.session_handler import
ResponsesSessionHandler,
StoredTurns,
)
from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex
from litellm.types.llms.anthropic import ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, AnthropicToolSearchToolRegex
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
ChatCompletionAssistantToolCall,
ChatCompletionAudioObject,
ChatCompletionCachedContent,
ChatCompletionFileObject,
@ -114,6 +121,7 @@ from litellm.types.responses.main import (
OutputImageGenerationCall,
OutputText,
OutputToolSearchCall,
web_search_call_tool_id,
)
from litellm.types.utils import (
ChatCompletionAnnotation,
@ -873,6 +881,29 @@ def _chain_session_metadata(chain_session: str, request_kwargs: Mapping[str, obj
return {key: {**_STR_KEY_DICT_ADAPTER.validate_python(metadata), CALLER_SESSION_ID_METADATA_KEY: chain_session}}
def _pending_claude_server_tool_call(search: ResponseFunctionWebSearch) -> ChatCompletionAssistantToolCall | None:
tool_id: Final = web_search_call_tool_id(search.id)
if search.status != "in_progress" or tool_id is None:
return None
if not tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX):
return None
match search.action:
case ActionSearch(query=str() as query):
return _server_tool_call(tool_id, "web_search", json.dumps({"query": query}))
case ActionOpenPage(url=str() as url):
return _server_tool_call(tool_id, "web_fetch", json.dumps({"url": url}))
case _:
return None
def _server_tool_call(tool_id: str, name: str, arguments: str) -> ChatCompletionAssistantToolCall:
return ChatCompletionAssistantToolCall(
id=tool_id,
type="function",
function=ChatCompletionToolCallFunctionChunk(name=name, arguments=arguments),
)
class LiteLLMCompletionResponsesConfig:
@staticmethod
def get_supported_openai_params(model: str) -> list:
@ -1778,10 +1809,22 @@ class LiteLLMCompletionResponsesConfig:
message_cache_control: Final = (
None if isinstance(content, list) else new_cache_control or previous_cache_control
)
new_server_tool_calls: Final = anthropic_server_tool_call_params(new_message)
server_tool_call_fields: Final = (
{
"provider_specific_fields": {
**(validated_object_mapping(last_message.get("provider_specific_fields")) or {}),
"server_tool_calls": [*anthropic_server_tool_call_params(last_message), *new_server_tool_calls],
}
}
if new_server_tool_calls
else {}
)
merged: Final = {
**{key: value for key, value in last_message.items() if key != "cache_control"},
"content": content,
**({} if message_cache_control is None else {"cache_control": message_cache_control}),
**server_tool_call_fields,
}
return cast( # cast-ok: preserves the assistant fields and content blocks
ChatCompletionAssistantMessage, merged
@ -2223,10 +2266,15 @@ class LiteLLMCompletionResponsesConfig:
content="Hosted web search: " + replay.model_dump_json(exclude_none=True),
)
]
hosted_search_text: Final = "Hosted web search: " + search.model_dump_json(exclude_none=True)
pending_server_tool_call: Final = _pending_claude_server_tool_call(search)
if pending_server_tool_call is None:
return [GenericChatCompletionMessage(role="assistant", content=hosted_search_text)]
return [
GenericChatCompletionMessage(
ChatCompletionResponseMessage(
role="assistant",
content="Hosted web search: " + search.model_dump_json(exclude_none=True),
content=hosted_search_text,
provider_specific_fields={"server_tool_calls": [pending_server_tool_call]},
)
]
if LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(input_item):
@ -2956,13 +3004,12 @@ class LiteLLMCompletionResponsesConfig:
all_chat_completion_tools: Final[list[ChatCompletionMessageToolCall]] = []
for choice in chat_completion_response.choices:
if isinstance(choice, Choices):
if choice.message.tool_calls:
all_chat_completion_tools.extend(choice.message.tool_calls)
for tool_call in choice.message.tool_calls:
TOOL_CALLS_CACHE.set_cache(
key=tool_call.id,
value=tool_call,
)
for tool_call in (*anthropic_server_tool_calls(choice.message), *(choice.message.tool_calls or ())):
all_chat_completion_tools.append(tool_call)
TOOL_CALLS_CACHE.set_cache(
key=tool_call.id,
value=tool_call,
)
request_tools: Final = responses_api_request.get("tools") if responses_api_request is not None else None
custom_tool_names: Final = extract_custom_tool_names(request_tools)
@ -3379,7 +3426,22 @@ class LiteLLMCompletionResponsesConfig:
for item in responses_output
]
return responses_output
server_tool_call_ids: Final = LiteLLMCompletionResponsesConfig._server_tool_call_ids(choices)
return [
item
for item in responses_output
if not (
getattr(item, "type", None) == "function_call"
and getattr(item, "call_id", None) in server_tool_call_ids
)
]
@staticmethod
def _server_tool_call_ids(choices: Sequence[Choices]) -> frozenset[str]:
server_tool_calls: Final = chain.from_iterable(
anthropic_server_tool_calls(choice.message) for choice in choices
)
return frozenset(tool_call.id for tool_call in server_tool_calls)
@staticmethod
def _extract_tool_result_output_items(

View file

@ -120,9 +120,12 @@ def _replays_provider_specific_fields(
client_tool_calls: tuple[ChatCompletionMessageToolCall, ...],
) -> bool:
from litellm.completion_extras.litellm_responses_transformation.transformation import carries_tool_search_items
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
return client_tool_calls != tool_calls or carries_tool_search_items(
getattr(message, "provider_specific_fields", None)
return (
client_tool_calls != tool_calls
or bool(anthropic_server_tool_call_params(message))
or carries_tool_search_items(getattr(message, "provider_specific_fields", None))
)

View file

@ -3,7 +3,12 @@ from collections.abc import Mapping, Sequence
from typing import Final, Literal, Optional, Union
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
from openai.types.responses.response_function_web_search import ActionSearchSource, ResponseFunctionWebSearch
from openai.types.responses.response_function_web_search import (
ActionOpenPage,
ActionSearch,
ActionSearchSource,
ResponseFunctionWebSearch,
)
from pydantic import ConfigDict, PrivateAttr
from typing_extensions import TypedDict
@ -42,14 +47,11 @@ class OutputFunctionToolCall(BaseLiteLLMOpenAIResponseObject):
phase: Phase = None
def build_web_search_call(
tool_id: str,
tool_input: object,
result: object,
status: Literal["in_progress", "searching", "completed", "failed"] | None = None,
) -> ResponseFunctionWebSearch:
HOSTED_WEB_TOOL_NAMES: Final = frozenset({"web_search", "web_fetch"})
def _web_search_action(tool_input: object, content: object) -> ActionSearch:
query: Final = tool_input.get("query", "") if isinstance(tool_input, Mapping) else ""
content: Final = result.get("content") if isinstance(result, Mapping) else None
result_items: Final = content if isinstance(content, Sequence) and not isinstance(content, (str, bytes)) else ()
sources: Final = [
ActionSearchSource(type="url", url=url)
@ -58,20 +60,62 @@ def build_web_search_call(
and item.get("type") == "web_search_result"
and isinstance((url := item.get("url")), str)
]
failed: Final = isinstance(content, Mapping) and content.get("type") == "web_search_tool_result_error"
return ResponseFunctionWebSearch(
id=f"ws_{tool_id}",
type="web_search_call",
status=status or ("failed" if failed else "completed"),
action={
"type": "search",
"query": query if isinstance(query, str) else "",
"queries": [query] if isinstance(query, str) and query else [],
"sources": sources,
},
return ActionSearch(
type="search",
query=query if isinstance(query, str) else "",
queries=[query] if isinstance(query, str) and query else [],
sources=sources,
)
def _web_fetch_action(tool_input: object) -> ActionOpenPage:
match tool_input:
case {"url": str() as url}:
return ActionOpenPage(type="open_page", url=url)
case _:
return ActionOpenPage(type="open_page", url=None)
def _result_content(result: object) -> object:
match result:
case {"content": object() as content}:
return content
case _:
return None
def _is_tool_result_error(content: object, tool_name: str) -> bool:
match content:
case {"type": str() as content_type}:
return content_type == f"{tool_name}_tool_result_error"
case _:
return False
WEB_SEARCH_CALL_ID_PREFIX: Final = "ws_"
def build_web_search_call(
tool_id: str,
tool_input: object,
result: object,
status: Literal["in_progress", "searching", "completed", "failed"] | None = None,
tool_name: str = "web_search",
) -> ResponseFunctionWebSearch:
content: Final = _result_content(result)
return ResponseFunctionWebSearch(
id=f"{WEB_SEARCH_CALL_ID_PREFIX}{tool_id}",
type="web_search_call",
status=status or ("failed" if _is_tool_result_error(content, tool_name) else "completed"),
action=_web_fetch_action(tool_input) if tool_name == "web_fetch" else _web_search_action(tool_input, content),
)
def web_search_call_tool_id(call_id: str) -> str | None:
tool_id: Final = call_id.removeprefix(WEB_SEARCH_CALL_ID_PREFIX)
return None if tool_id == call_id else tool_id
class OutputImageGenerationCall(BaseLiteLLMOpenAIResponseObject):
"""An image generation call output"""

View file

@ -110,6 +110,41 @@ def test_output_tool_calls_use_the_datadog_tool_call_schema(logger: DataDogLLMOb
assert "function" not in message["tool_calls"][0]
def test_claude_server_tool_calls_are_reported_before_client_tool_calls(logger: DataDogLLMObsLogger) -> None:
payload = build(
logger,
response_message={
"role": "assistant",
"content": None,
"tool_calls": [ASSISTANT_TOOL_CALL],
"provider_specific_fields": {
"server_tool_calls": [
{
"id": "srvtoolu_01Search",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "paris weather"}'},
}
]
},
},
)
assert payload["meta"]["output"]["messages"][0]["tool_calls"] == [
{
"name": "web_search",
"arguments": {"query": "paris weather"},
"tool_id": "srvtoolu_01Search",
"type": "function",
},
{
"name": "get_weather",
"arguments": {"city": "Paris", "unit": "c"},
"tool_id": "call_abc123",
"type": "function",
},
]
def test_tool_call_identifiers_that_are_not_strings_are_blanked_not_stringified(logger: DataDogLLMObsLogger) -> None:
payload = build(
logger,

View file

@ -174,6 +174,45 @@ def test_openinference_output_tool_calls_preserve_calls_in_attributes_and_value(
]
def test_openinference_output_lists_claude_server_tool_calls_before_client_tool_calls():
data: Final = _llm_call(
choices_out=(
{
"finish_reason": "tool_calls",
"message": {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_paris",
"type": "function",
"function": {"name": "lookup_weather", "arguments": '{"city": "Paris"}'},
}
],
"provider_specific_fields": {
"server_tool_calls": [
{
"id": "srvtoolu_01Search",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "paris weather"}'},
}
]
},
},
},
)
)
attrs: Final = OpenInferenceMapper().map(data)
assert {key: value for key, value in attrs.items() if ".tool_calls." in key} == {
"llm.output_messages.0.message.tool_calls.0.tool_call.id": "srvtoolu_01Search",
"llm.output_messages.0.message.tool_calls.0.tool_call.function.name": "web_search",
"llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": '{"query": "paris weather"}',
"llm.output_messages.0.message.tool_calls.1.tool_call.id": "call_paris",
"llm.output_messages.0.message.tool_calls.1.tool_call.function.name": "lookup_weather",
"llm.output_messages.0.message.tool_calls.1.tool_call.function.arguments": '{"city": "Paris"}',
}
def test_openinference_responses_tool_calls_are_emitted_as_output_attributes():
data: Final = LLMCallSpanData.from_standard_logging_payload(
_responses_payload(

View file

@ -15,7 +15,9 @@ import litellm
from litellm._uuid import uuid
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
from litellm.llms.anthropic.chat.handler import ModelResponseIterator, _envelope_verdict, make_call, make_sync_call
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
from litellm.llms.base_llm.base_utils import response_format_schema, unwrap_response_format_envelope
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.types.llms.openai import (
@ -23,7 +25,7 @@ from litellm.types.llms.openai import (
ChatCompletionToolCallFunctionChunk,
)
from litellm.types.responses.main import OutputCodeInterpreterCall
from litellm.types.utils import ChatCompletionDeltaToolCall, ModelResponseStream
from litellm.types.utils import ChatCompletionDeltaToolCall, ChatCompletionMessageToolCall, Choices, ModelResponseStream
from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE
@ -1780,19 +1782,113 @@ def test_web_search_tool_result_no_extra_tool_calls():
for tc in parsed.choices[0].delta.tool_calls:
tool_calls_emitted.append(tc)
# Should have exactly 2 tool calls:
# 1. From content_block_start (server_tool_use) with id and name
# 2. From content_block_delta with the actual query
assert len(tool_calls_emitted) == 2, f"Expected 2 tool calls, got {len(tool_calls_emitted)}"
assert tool_calls_emitted == []
# First tool call should have the id and name
assert tool_calls_emitted[0]["id"] == "srvtoolu_01ABC123"
assert tool_calls_emitted[0]["function"]["name"] == "web_search"
# Second tool call should have the query arguments
assert tool_calls_emitted[1]["function"]["arguments"] == '{"query": "test"}'
def _server_tool_input_delta(fragment: str) -> dict[str, object]:
return {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": fragment}}
# The {} chunks from web_search_tool_result should NOT have been emitted as tool calls
def _server_search_then_answer(*partial_json: str) -> tuple[dict[str, object], ...]:
return (
{
"type": "message_start",
"message": {"id": "msg_01", "model": "claude-opus-5-5", "usage": {"input_tokens": 9}},
},
{
"type": "content_block_start",
"index": 0,
"content_block": {"type": "server_tool_use", "id": "srvtoolu_01Web", "name": "web_search", "input": {}},
},
*(_server_tool_input_delta(fragment) for fragment in partial_json),
{"type": "content_block_stop", "index": 0},
{
"type": "content_block_start",
"index": 1,
"content_block": {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_01Web", "content": []},
},
{"type": "content_block_stop", "index": 1},
{"type": "content_block_start", "index": 2, "content_block": {"type": "text", "text": ""}},
{"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": "Paris is in France."}},
{"type": "content_block_stop", "index": 2},
)
_SERVER_SEARCH_THEN_ANSWER: Final = _server_search_then_answer('{"query": "Paris"}')
_CLIENT_TOOL_CALL: Final[tuple[dict[str, object], ...]] = (
{
"type": "content_block_start",
"index": 3,
"content_block": {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {}},
},
{
"type": "content_block_delta",
"index": 3,
"delta": {"type": "input_json_delta", "partial_json": '{"city": "Paris"}'},
},
{"type": "content_block_stop", "index": 3},
)
def _message_delta(stop_reason: str) -> dict[str, object]:
return {"type": "message_delta", "delta": {"stop_reason": stop_reason}, "usage": {"output_tokens": 20}}
_PARIS_SEARCH: Final = ("srvtoolu_01Web", "web_search", '{"query": "Paris"}')
@pytest.mark.parametrize(
("events", "expected_tool_calls", "expected_finish_reason", "expected_server_tool_call"),
[
pytest.param(
(*_SERVER_SEARCH_THEN_ANSWER, _message_delta("end_turn")), [], "stop", _PARIS_SEARCH, id="server_tool_only"
),
pytest.param(
(*_SERVER_SEARCH_THEN_ANSWER, *_CLIENT_TOOL_CALL, _message_delta("tool_use")),
[("toolu_01Weather", "get_weather", '{"city": "Paris"}')],
"tool_calls",
_PARIS_SEARCH,
id="server_tool_then_client_tool",
),
pytest.param(
(*_server_search_then_answer('{"que', 'ry": "Pa', 'ris"}'), _message_delta("end_turn")),
[],
"stop",
_PARIS_SEARCH,
id="server_tool_input_split_across_deltas",
),
pytest.param(
(*_server_search_then_answer(), _message_delta("end_turn")),
[],
"stop",
("srvtoolu_01Web", "web_search", "{}"),
id="server_tool_with_empty_input",
),
],
)
def test_streamed_server_tool_use_is_not_returned_as_a_client_tool_call(
events: tuple[dict[str, object], ...],
expected_tool_calls: list[tuple[str, str, str]],
expected_finish_reason: str,
expected_server_tool_call: tuple[str, str, str],
):
choice: Final = _assemble_stream(events).choices[0]
assert isinstance(choice, Choices)
assert (choice.message.tool_calls is None) == (not expected_tool_calls)
assert [
(call.id, call.function.name, call.function.arguments)
for call in choice.message.tool_calls or ()
if isinstance(call, ChatCompletionMessageToolCall)
] == expected_tool_calls
assert choice.finish_reason == expected_finish_reason
assert choice.message.content == "Paris is in France."
assert [
(call["id"], call["function"].get("name"), call["function"].get("arguments"))
for call in anthropic_server_tool_call_params(choice.message)
] == [expected_server_tool_call]
web_search_results: Final = message_field(choice.message.provider_specific_fields, "web_search_results")
assert [message_field(result, "tool_use_id") for result in parts_of(web_search_results)] == ["srvtoolu_01Web"]
def test_current_content_block_type_tracking():

View file

@ -1,7 +1,8 @@
import copy
import json
from collections.abc import Mapping
from itertools import chain
from typing import Final
from typing import Final, cast
from unittest.mock import MagicMock, patch
import httpx
@ -22,8 +23,13 @@ from litellm.constants import (
)
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
from litellm.llms.anthropic.chat import ModelResponseIterator
from litellm.llms.anthropic.common_utils import process_anthropic_headers
from litellm.llms.anthropic.common_utils import (
anthropic_server_tool_call_params,
process_anthropic_headers,
validated_object_mapping,
)
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema
from litellm.llms.anthropic.messages.native.transformation import (
@ -38,8 +44,8 @@ from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation im
VertexAIAnthropicConfig,
)
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES
from litellm.types.llms.openai import ChatCompletionToolCallFunctionChunk
from litellm.types.utils import ChatCompletionToolCallChunk, ModelResponse, ServerToolUse, Usage
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallFunctionChunk
from litellm.types.utils import ChatCompletionToolCallChunk, Choices, Message, ModelResponse, ServerToolUse, Usage
from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE
@ -3509,21 +3515,17 @@ def test_code_execution_tool_results_extraction():
prefix_prompt=None,
)
# Verify tool calls are present
assert transformed_response.choices[0].message.tool_calls is not None
assert len(transformed_response.choices[0].message.tool_calls) == 2
assert transformed_response.choices[0].message.tool_calls is None
# Verify first tool call
assert transformed_response.choices[0].message.tool_calls[0].id == "srvtoolu_01ABC"
assert transformed_response.choices[0].message.tool_calls[0].function.name == "bash_code_execution"
# Verify second tool call
assert transformed_response.choices[0].message.tool_calls[1].id == "srvtoolu_01DEF"
assert transformed_response.choices[0].message.tool_calls[1].function.name == "text_editor_code_execution"
# Verify tool results are in provider_specific_fields
provider_fields = transformed_response.choices[0].message.provider_specific_fields
assert provider_fields is not None
server_tool_calls: Final = anthropic_server_tool_call_params(transformed_response.choices[0].message)
assert [(call["id"], call["function"].get("name")) for call in server_tool_calls] == [
("srvtoolu_01ABC", "bash_code_execution"),
("srvtoolu_01DEF", "text_editor_code_execution"),
]
# Verify tool results are in provider_specific_fields
assert "tool_results" in provider_fields
assert provider_fields["tool_results"] is not None
assert len(provider_fields["tool_results"]) == 2
@ -3633,7 +3635,7 @@ def _tool_search_turn(tool_references: list[str], last_block: dict[str, object])
]
def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str):
def _parsed_choice(content: list[dict[str, object]], stop_reason: str) -> Choices:
anthropic_response: Final = {
"id": "msg_01XYZ",
"type": "message",
@ -3648,7 +3650,7 @@ def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str
raw_response.json.return_value = anthropic_response
raw_response.status_code = 200
raw_response.headers = {}
return (
choice: Final = (
AnthropicConfig()
.transform_parsed_response(
completion_response=anthropic_response,
@ -3658,14 +3660,19 @@ def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str
prefix_prompt=None,
)
.choices[0]
.message
)
assert isinstance(choice, Choices)
return choice
def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str) -> Message:
return _parsed_choice(content, stop_reason).message
_WEATHER_CALL: Final = {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}}
def _web_search_turn(last_block: dict[str, object]) -> list[dict[str, object]]:
def _web_search_turn(last_block: Mapping[str, object]) -> list[dict[str, object]]:
return [
{"type": "server_tool_use", "id": "srvtoolu_01Web", "name": "web_search", "input": {"query": "Paris facts"}},
{
@ -3681,7 +3688,7 @@ def _web_search_turn(last_block: dict[str, object]) -> list[dict[str, object]]:
}
],
},
last_block,
dict(last_block),
]
@ -3728,8 +3735,114 @@ def test_server_tool_result_round_trips_right_after_its_call(
assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"]
def _block_ids(message: dict[str, object], block_type: str, id_key: str) -> list[object]:
return sorted(block[id_key] for block in message["content"] if block["type"] == block_type)
_FETCH_STILL_WAITING_ON_ITS_RESULT: Final = {
"type": "server_tool_use",
"id": "srvtoolu_01Fetch",
"name": "web_fetch",
"input": {"url": "https://example.com/"},
}
@pytest.mark.parametrize("modify_params", [False, True])
def test_server_call_still_waiting_on_its_result_is_replayed_as_a_server_call(
modify_params: bool, monkeypatch: pytest.MonkeyPatch
):
content: Final = [_FETCH_STILL_WAITING_ON_ITS_RESULT, _WEATHER_CALL]
message: Final = _parsed_assistant_message(content, "tool_use")
monkeypatch.setattr(litellm, "modify_params", modify_params)
replayed: Final = _replayed(
{"role": "user", "content": "Fetch example.com, then tell me the weather in Paris"},
message.model_dump(),
{"role": "tool", "tool_call_id": "toolu_01Weather", "content": "18C"},
)
assert replayed[1]["content"] == content
assert [turn["role"] for turn in replayed] == ["user", "assistant", "user"]
assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"]
_PARIS_TEXT: Final = "Paris is the capital of France."
_PARIS_ANSWER: Final[Mapping[str, str]] = {"type": "text", "text": _PARIS_TEXT}
@pytest.mark.parametrize(
("content", "stop_reason", "expected_tool_call_ids", "expected_finish_reason"),
[
pytest.param(_web_search_turn(_PARIS_ANSWER), "end_turn", [], "stop", id="web_search_then_answer"),
pytest.param(
_web_search_turn(_WEATHER_CALL), "tool_use", ["toolu_01Weather"], "tool_calls", id="web_search_then_tool"
),
pytest.param([_WEATHER_CALL], "tool_use", ["toolu_01Weather"], "tool_calls", id="client_tool_only"),
],
)
def test_server_tool_use_is_not_returned_as_a_client_tool_call(
content: list[dict[str, object]],
stop_reason: str,
expected_tool_call_ids: list[str],
expected_finish_reason: str,
):
choice: Final = _parsed_choice(content, stop_reason)
assert (choice.message.tool_calls is None) == (not expected_tool_call_ids)
assert [call.id for call in choice.message.tool_calls or ()] == expected_tool_call_ids
assert choice.finish_reason == expected_finish_reason
assert [
(call["id"], call["function"].get("name"), call["function"].get("arguments"))
for call in anthropic_server_tool_call_params(choice.message)
] == [
(block["id"], block["name"], json.dumps(block["input"]))
for block in content
if block["type"] == "server_tool_use"
]
if not expected_tool_call_ids:
assert choice.message.content == _PARIS_TEXT
def _replayed(*messages: object):
return anthropic_messages_pt(
messages=cast(list[AllMessageValues], list(messages)), # cast-ok: replays history the way clients send it back
model="claude-opus-5-5",
llm_provider="anthropic",
)
def test_web_search_answer_round_trips_without_any_client_tool_call():
message: Final = _parsed_assistant_message(_web_search_turn(_PARIS_ANSWER), "end_turn")
replayed: Final = _replayed(
{"role": "user", "content": "Tell me about Paris"},
message.model_dump(),
{"role": "user", "content": "And Lyon?"},
)
assert _block_ids(replayed[1], "server_tool_use", "id") == ["srvtoolu_01Web"]
assert _block_ids(replayed[1], "web_search_tool_result", "tool_use_id") == ["srvtoolu_01Web"]
assert [
message_field(block, "text")
for block in parts_of(replayed[1]["content"])
if message_field(block, "type") == "text"
] == [_PARIS_TEXT]
def _block_ids(message: Mapping[str, object], block_type: str, id_key: str) -> list[str]:
return sorted(
block_id
for block in parts_of(message["content"])
if message_field(block, "type") == block_type and isinstance(block_id := message_field(block, id_key), str)
)
def _as_returned_before_server_tool_calls_moved(message: Message) -> Mapping[str, object]:
dumped: Final[Mapping[str, object]] = message.model_dump()
provider_fields: Final = validated_object_mapping(dumped.get("provider_specific_fields")) or {}
return {
**dumped,
"tool_calls": [*anthropic_server_tool_call_params(message), *parts_of(dumped.get("tool_calls"))],
"provider_specific_fields": {
key: value for key, value in provider_fields.items() if key != "server_tool_calls"
},
}
_SEARCH_THAT_FOUND_NOTHING: Final = _tool_search_turn(
@ -3764,21 +3877,33 @@ def test_replayed_search_that_found_nothing_answers_every_tool_use_and_ends_on_a
server_tool_use_ids: list[str],
monkeypatch: pytest.MonkeyPatch,
):
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
message: Final = _parsed_assistant_message(_SEARCH_THAT_FOUND_NOTHING, "end_turn")
monkeypatch.setattr(litellm, "modify_params", modify_params)
replayed: Final = anthropic_messages_pt(
messages=[{"role": "user", "content": "Find a tool and use it"}, message.model_dump(), *follow_ups],
model="claude-opus-5-5",
llm_provider="anthropic",
replayed: Final = _replayed(
{"role": "user", "content": "Find a tool and use it"},
_as_returned_before_server_tool_calls_moved(message),
*follow_ups,
)
assert [turn["role"] for turn in replayed] == ["user", "assistant", "user"]
assert _block_ids(replayed[2], "tool_result", "tool_use_id") == _block_ids(replayed[1], "tool_use", "id")
assert _block_ids(replayed[1], "server_tool_use", "id") == server_tool_use_ids
def test_replayed_search_that_found_nothing_keeps_its_result_when_the_call_never_reached_the_client(
monkeypatch: pytest.MonkeyPatch,
):
message: Final = _parsed_assistant_message(_SEARCH_THAT_FOUND_NOTHING, "end_turn")
monkeypatch.setattr(litellm, "modify_params", True)
replayed: Final = _replayed({"role": "user", "content": "Find a tool and use it"}, message.model_dump())
assert [turn["role"] for turn in replayed] == ["user", "assistant"]
assert _block_ids(replayed[1], "server_tool_use", "id") == ["srvtoolu_01Search"]
assert _block_ids(replayed[1], "tool_search_tool_result", "tool_use_id") == ["srvtoolu_01Search"]
assert _block_ids(replayed[1], "tool_use", "id") == []
_CODE_EXECUTION_BLOCKS: Final = [
{
"type": "server_tool_use",

View file

@ -80,6 +80,59 @@ def test_native_compaction_response_roundtrip(text: str | None, count: int, expe
assert result["stop_reason"] == expected_stop
_SERVER_AND_CLIENT_TOOL_TURN: Final[tuple[dict[str, object], ...]] = (
{
"type": "server_tool_use",
"id": "srvtoolu_01Fetch",
"name": "web_fetch",
"input": {"url": "https://example.com/"},
},
{
"type": "web_fetch_tool_result",
"tool_use_id": "srvtoolu_01Fetch",
"content": {"type": "web_fetch_result", "url": "https://example.com/"},
},
{"type": "server_tool_use", "id": "srvtoolu_02Search", "name": "web_search", "input": {"query": "Paris"}},
{"type": "text", "text": "The page is titled Example."},
{"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}},
)
def test_claude_server_tool_blocks_survive_the_round_trip_through_chat_completions() -> None:
from unittest.mock import MagicMock
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
anthropic_response: Final = {
"id": "msg_01Fetch",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": list(_SERVER_AND_CLIENT_TOOL_TURN),
"stop_reason": "tool_use",
"stop_sequence": None,
"usage": {"input_tokens": 100, "output_tokens": 50},
}
raw_response: Final = MagicMock(spec=httpx.Response)
raw_response.json.return_value = anthropic_response
raw_response.status_code = 200
raw_response.headers = {}
chat_response: Final = AnthropicConfig().transform_parsed_response(
completion_response=anthropic_response,
raw_response=raw_response,
model_response=ModelResponse(),
json_mode=False,
prefix_prompt=None,
)
result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(chat_response)
assert [(block["type"], block.get("id") or block.get("tool_use_id")) for block in result["content"]] == [
(block["type"], block.get("id") or block.get("tool_use_id")) for block in _SERVER_AND_CLIENT_TOOL_TURN
]
assert result["stop_reason"] == "tool_use"
def test_translate_chat_refusal_to_anthropic_response():
response = ModelResponse(
id="chatcmpl-refusal",

View file

@ -144,6 +144,42 @@ class MockMisalignedToolCallGuardrail(CustomGuardrail):
)
def _server_tool_call(call_id: str, name: str, arguments: Mapping[str, object]) -> dict[str, object]:
return {"id": call_id, "type": "function", "function": {"name": name, "arguments": json.dumps(arguments)}}
def _tool_call_ids_names_and_arguments(tool_calls) -> list[tuple[str, str, str]]:
return [
(tool_call["id"], tool_call["function"]["name"], tool_call["function"]["arguments"])
for tool_call in tool_calls
]
class QueryRedactingGuardrail(CustomGuardrail):
"""Returns copied tool calls with any ``query`` argument redacted, leaving other calls as they came."""
async def apply_guardrail(
self,
inputs: GenericGuardrailAPIInputs,
request_data: dict,
input_type: Literal["request", "response"],
logging_obj: Optional[Any] = None,
) -> GenericGuardrailAPIInputs:
def redacted(tool_call):
arguments = json.loads(tool_call["function"]["arguments"])
if "query" not in arguments:
return tool_call
return {
**tool_call,
"function": {**tool_call["function"], "arguments": json.dumps({**arguments, "query": "[REDACTED]"})},
}
return GenericGuardrailAPIInputs(
texts=inputs.get("texts", []),
tool_calls=[redacted(tool_call) for tool_call in inputs.get("tool_calls", [])],
)
class TestOpenAIChatCompletionsHandlerToolsInput:
"""Test input processing with tools (function definitions)"""
@ -906,6 +942,92 @@ class TestOpenAIChatCompletionsHandlerToolCallsOutput:
assert tool_calls[0].function.arguments == first_args
assert tool_calls[1].function.arguments == second_args
@pytest.mark.asyncio
async def test_output_guardrail_sees_a_response_that_only_carries_server_tool_calls(self):
handler = OpenAIChatCompletionsHandler()
guardrail = TestOpenAIChatCompletionsHandlerStreamingOutput._recording_guardrail()
response = ModelResponse(
model="claude-sonnet-4-5",
choices=[
Choices(
finish_reason="stop",
index=0,
message=Message(
content=None,
role="assistant",
provider_specific_fields={
"server_tool_calls": [_server_tool_call("srvtoolu_01Search", "web_search", {"query": "x"})]
},
),
)
],
)
await handler.process_output_response(response, guardrail)
assert _tool_call_ids_names_and_arguments(guardrail.seen_inputs[-1]["tool_calls"]) == [
("srvtoolu_01Search", "web_search", '{"query": "x"}')
]
@pytest.mark.asyncio
async def test_output_guardrail_rewrite_of_a_server_tool_call_lands_in_server_tool_calls_only(self):
handler = OpenAIChatCompletionsHandler()
weather_args = json.dumps({"city": "Paris", "query": "forecast"})
response = ModelResponse(
model="claude-sonnet-4-5",
choices=[
Choices(
finish_reason="tool_calls",
index=0,
message=Message(
content="Looking it up",
role="assistant",
tool_calls=[
ChatCompletionMessageToolCall(
id="call_1",
type="function",
function=Function(name="get_weather", arguments=json.dumps({"city": "Paris"})),
)
],
provider_specific_fields={
"server_tool_calls": [
_server_tool_call("srvtoolu_01Search", "web_search", {"query": "jane@example.com"})
]
},
),
),
Choices(
finish_reason="tool_calls",
index=1,
message=Message(
content=None,
role="assistant",
tool_calls=[
ChatCompletionMessageToolCall(
id="call_2",
type="function",
function=Function(name="get_weather", arguments=weather_args),
)
],
),
),
],
)
await handler.process_output_response(response, QueryRedactingGuardrail(guardrail_name="redact"))
first, second = response.choices
assert _tool_call_ids_names_and_arguments(first.message.provider_specific_fields["server_tool_calls"]) == [
("srvtoolu_01Search", "web_search", '{"query": "[REDACTED]"}')
]
assert [(call.id, call.function.arguments) for call in first.message.tool_calls] == [
("call_1", '{"city": "Paris"}')
]
assert [(call.id, call.function.arguments) for call in second.message.tool_calls] == [
("call_2", '{"city": "Paris", "query": "[REDACTED]"}')
]
assert not (second.message.provider_specific_fields or {}).get("server_tool_calls")
class MockPassThroughGuardrail(CustomGuardrail):
"""Mock guardrail that passes through without blocking - for testing streaming fallback behavior"""
@ -1462,6 +1584,61 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
assert chunks[0].choices[0].delta.content == "hello [MASKED]"
assert chunks[1].choices[0].delta.content in (None, "")
@staticmethod
def _ended_server_tool_call_stream_chunks() -> list:
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
def chunk(delta: Delta, finish_reason: Optional[str] = None) -> ModelResponseStream:
return ModelResponseStream(
id="chatcmpl-123",
created=1234567890,
model="claude-sonnet-4-5",
object="chat.completion.chunk",
choices=[StreamingChoices(index=0, delta=delta, finish_reason=finish_reason)],
)
search = {**_server_tool_call("srvtoolu_01Search", "web_search", {"query": "jane@example.com"}), "index": 0}
return [
chunk(Delta(content="Searching")),
chunk(Delta(content=None, provider_specific_fields={"server_tool_calls": [search]})),
chunk(Delta(content=None), finish_reason="stop"),
]
@pytest.mark.asyncio
async def test_ended_stream_guardrail_sees_streamed_server_tool_calls(self):
handler = OpenAIChatCompletionsHandler()
guardrail = self._recording_guardrail()
await handler.process_output_streaming_response(
responses_so_far=self._ended_server_tool_call_stream_chunks(),
guardrail_to_apply=guardrail,
litellm_logging_obj=None,
deliver_ended_stream_rewrites=True,
)
assert _tool_call_ids_names_and_arguments(guardrail.seen_inputs[-1]["tool_calls"]) == [
("srvtoolu_01Search", "web_search", '{"query": "jane@example.com"}')
]
@pytest.mark.asyncio
async def test_deliver_ended_stream_rewrites_writes_server_tool_call_arguments_back_into_chunks(self):
handler = OpenAIChatCompletionsHandler()
chunks = self._ended_server_tool_call_stream_chunks()
await handler.process_output_streaming_response(
responses_so_far=chunks,
guardrail_to_apply=QueryRedactingGuardrail(guardrail_name="redact"),
litellm_logging_obj=None,
deliver_ended_stream_rewrites=True,
)
delta = chunks[1].choices[0].delta
assert _tool_call_ids_names_and_arguments(delta.provider_specific_fields["server_tool_calls"]) == [
("srvtoolu_01Search", "web_search", '{"query": "[REDACTED]"}')
]
assert delta.tool_calls is None
assert chunks[0].choices[0].delta.content == "Searching"
class TestUndecoratedGuardrailIsRecorded:
"""LIT-5983 regression: the handler calls apply_guardrail bare, so a custom guardrail
@ -2323,6 +2500,29 @@ class TestStreamingScanKey:
assert len(ended_key.tool_calls) == 1 and "run_shell" in ended_key.tool_calls[0]
assert ended_key.tool_calls_in_flight is False
def test_server_tool_calls_hold_the_window_and_enter_the_key_once_the_stream_has_ended(self):
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
handler = OpenAIChatCompletionsHandler()
server_chunk = ModelResponseStream(
choices=[
StreamingChoices(
index=0,
delta=Delta(
content=None,
provider_specific_fields={
"server_tool_calls": [_server_tool_call("srvtoolu_01Search", "web_search", {"query": "x"})]
},
),
)
]
)
open_key = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk])
ended_key = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk, self._chunk(None, "stop")])
assert open_key.tool_calls_in_flight is True
assert len(ended_key.tool_calls) == 1 and "web_search" in ended_key.tool_calls[0]
assert ended_key != open_key
def test_text_after_the_first_choice_finishes_still_changes_the_key(self):
handler = OpenAIChatCompletionsHandler()
first_done = [self._chunk("a", index=0), self._chunk("b", finish_reason="stop", index=0)]

View file

@ -23,6 +23,7 @@ from litellm.proxy.db.spend_log_tool_index import (
)
from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import ModelResponse
from tests.unit.proxy.db.fake_prisma_engine import engine_call
@ -204,6 +205,36 @@ class TestResponseToolCallNames:
def test_blank_names_are_dropped(self):
assert response_tool_call_names(_response_with_tool_calls(" ", "real_tool")) == ("real_tool",)
@pytest.mark.parametrize("as_object", [False, True], ids=["dict", "model_response"])
def test_server_tool_calls_of_every_choice_are_named_before_client_tool_calls(self, as_object: bool):
def tool_call(call_id: str, name: str) -> dict[str, object]:
return {"id": call_id, "type": "function", "function": {"name": name, "arguments": "{}"}}
response: dict[str, object] = {
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": None,
"tool_calls": [tool_call("call_1", "get_weather")],
"provider_specific_fields": {"server_tool_calls": [tool_call("srvtoolu_01", "web_search")]},
},
},
{
"index": 1,
"message": {
"role": "assistant",
"content": "done",
"provider_specific_fields": {"server_tool_calls": [tool_call("srvtoolu_02", "web_fetch")]},
},
},
]
}
completion_response = ModelResponse(**response) if as_object else response
assert response_tool_call_names(completion_response) == ("web_search", "web_fetch", "get_weather")
def test_responses_api_output_function_calls(self):
# Regression: /v1/responses carries invocations in output[] items of
# type function_call, not in choices; they must reach the rollup.

View file

@ -27,11 +27,21 @@ from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import (
from litellm.types.utils import (
ChatCompletionMessageToolCall,
Choices,
Delta,
ModelResponse,
ModelResponseStream,
StreamingChoices,
)
_SERVER_WEB_SEARCH_CALL: Final = {
"id": "srvtoolu_01Search",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "weather in Paris"}'},
}
_ALLOWED_BASH_CALL: Final = {"id": "toolu_01Bash", "type": "function", "function": {"name": "Bash", "arguments": "{}"}}
class TestToolPermissionGuardrail:
"""Test class for Tool Permission Guardrail functionality"""
@ -952,6 +962,71 @@ class TestToolPermissionGuardrail:
assert "Permission denied" in choice.message.content
def _server_search_response(self) -> ModelResponse:
return ModelResponse(
choices=[
Choices(
finish_reason="tool_calls",
message={
"content": "Searching.",
"tool_calls": [_ALLOWED_BASH_CALL],
"provider_specific_fields": {"server_tool_calls": [_SERVER_WEB_SEARCH_CALL]},
},
)
]
)
@pytest.mark.asyncio
async def test_denied_server_tool_call_is_blocked(self):
with patch.object(self.guardrail, "should_run_guardrail", return_value=True):
with pytest.raises(GuardrailRaisedException):
await self.guardrail.async_post_call_success_hook(
data={}, user_api_key_dict=UserAPIKeyAuth(), response=self._server_search_response()
)
@pytest.mark.asyncio
async def test_denied_server_tool_call_in_a_chat_stream_is_blocked(self):
async def _stream():
yield ModelResponseStream(
choices=[
StreamingChoices(
index=0,
delta=Delta(
content="Searching.",
provider_specific_fields={"server_tool_calls": [{**_SERVER_WEB_SEARCH_CALL, "index": 0}]},
),
finish_reason="stop",
)
]
)
with patch.object(self.guardrail, "should_run_guardrail", return_value=True):
with pytest.raises(GuardrailRaisedException):
async for _ in self.guardrail.async_post_call_streaming_iterator_hook(
user_api_key_dict=UserAPIKeyAuth(), response=_stream(), request_data={}
):
pass
@pytest.mark.asyncio
async def test_rewrite_mode_removes_the_denied_server_tool_call(self):
rewriting: Final = ToolPermissionGuardrail(
guardrail_name="test-tool-permission",
rules=self.test_rules,
default_action="deny",
on_disallowed_action="rewrite",
)
with patch.object(rewriting, "should_run_guardrail", return_value=True):
result: Final = await rewriting.async_post_call_success_hook(
data={}, user_api_key_dict=UserAPIKeyAuth(), response=self._server_search_response()
)
message: Final = result.choices[0].message
assert message.provider_specific_fields["server_tool_calls"] == []
assert [tool_call.id for tool_call in message.tool_calls] == [_ALLOWED_BASH_CALL["id"]]
assert "Permission denied" in message.content
class TestToolPermissionGuardrailIntegration:
"""Integration tests for Tool Permission Guardrail"""

View file

@ -33,7 +33,7 @@ from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
from litellm.types.llms.base import HiddenParams
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse
from litellm.types.responses.main import OutputToolSearchCall, build_web_search_call
from litellm.types.utils import (
ChatCompletionMessageToolCall,
@ -4993,6 +4993,159 @@ def test_function_call_tool_id_falls_back_to_unique_id_for_degenerate_call_id():
assert convert(openai)["id"] == "call_tokyo"
_WEATHER_TOOL_USE: Final = {
"type": "tool_use",
"id": "toolu_01Weather",
"name": "get_weather",
"input": {"city": "Paris"},
}
_FETCHED_URL: Final = "https://example.com/"
_BLOCKED_URL: Final = "https://blocked.invalid/"
_ANTHROPIC_WEB_FETCH_TURN: Final[tuple[dict[str, object], ...]] = (
{"type": "server_tool_use", "id": "srvtoolu_01Fetch", "name": "web_fetch", "input": {"url": _FETCHED_URL}},
{
"type": "web_fetch_tool_result",
"tool_use_id": "srvtoolu_01Fetch",
"content": {
"type": "web_fetch_result",
"url": _FETCHED_URL,
"content": {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "Example"}},
},
},
{"type": "server_tool_use", "id": "srvtoolu_02Fetch", "name": "web_fetch", "input": {"url": _BLOCKED_URL}},
{
"type": "web_fetch_tool_result",
"tool_use_id": "srvtoolu_02Fetch",
"content": {"type": "web_fetch_tool_result_error", "error_code": "url_not_allowed"},
},
{"type": "text", "text": "The page is titled Example."},
_WEATHER_TOOL_USE,
)
def _anthropic_turn_on_responses(content: Sequence[Mapping[str, object]]) -> ResponsesAPIResponse:
from unittest.mock import MagicMock
import httpx
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
anthropic_response: Final = {
"id": "msg_01Fetch",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": list(content),
"stop_reason": "tool_use",
"stop_sequence": None,
"usage": {"input_tokens": 100, "output_tokens": 50},
}
raw_response: Final = MagicMock(spec=httpx.Response)
raw_response.json.return_value = anthropic_response
raw_response.status_code = 200
raw_response.headers = {}
chat_response: Final = AnthropicConfig().transform_parsed_response(
completion_response=anthropic_response,
raw_response=raw_response,
model_response=ModelResponse(),
json_mode=False,
prefix_prompt=None,
)
return LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(
request_input="Use the web and check the weather in Paris.",
responses_api_request={
"tools": [
{"type": "web_search_20250305", "name": "web_search"},
{"type": "web_fetch_20250910", "name": "web_fetch"},
{"type": "function", "name": "get_weather", "parameters": {"type": "object"}},
]
},
chat_completion_response=chat_response,
)
def test_anthropic_web_fetch_is_a_hosted_open_page_call_on_responses() -> None:
response: Final = _anthropic_turn_on_responses(_ANTHROPIC_WEB_FETCH_TURN)
assert [
(item.id, item.status, item.action.model_dump()) for item in response.output if item.type == "web_search_call"
] == [
("ws_srvtoolu_01Fetch", "completed", {"type": "open_page", "url": _FETCHED_URL}),
("ws_srvtoolu_02Fetch", "failed", {"type": "open_page", "url": _BLOCKED_URL}),
]
assert [item.name for item in response.output if item.type == "function_call"] == ["get_weather"]
@pytest.mark.parametrize(
("server_tool_use", "expected_action"),
[
(
{
"type": "server_tool_use",
"id": "srvtoolu_01Pending",
"name": "web_fetch",
"input": {"url": _FETCHED_URL},
},
{"type": "open_page", "url": _FETCHED_URL},
),
(
{"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_search", "input": {"query": "python"}},
{"type": "search", "query": "python"},
),
],
)
def test_anthropic_server_web_call_without_a_result_is_an_in_progress_hosted_call_on_responses(
server_tool_use: Mapping[str, object], expected_action: Mapping[str, object]
) -> None:
response: Final = _anthropic_turn_on_responses((server_tool_use, _WEATHER_TOOL_USE))
web_calls: Final = [item for item in response.output if item.type == "web_search_call"]
assert [(item.id, item.status) for item in web_calls] == [("ws_srvtoolu_01Pending", "in_progress")]
assert expected_action.items() <= web_calls[0].action.model_dump().items()
assert [item.name for item in response.output if item.type == "function_call"] == ["get_weather"]
def test_anthropic_code_execution_is_never_a_client_function_call_on_responses() -> None:
response: Final = _anthropic_turn_on_responses(
(
{
"type": "server_tool_use",
"id": "srvtoolu_01Bash",
"name": "bash_code_execution",
"input": {"command": "ls"},
},
{
"type": "bash_code_execution_tool_result",
"tool_use_id": "srvtoolu_01Bash",
"content": {
"type": "bash_code_execution_result",
"stdout": "notes.txt\n",
"stderr": "",
"return_code": 0,
},
},
{
"type": "server_tool_use",
"id": "srvtoolu_02Edit",
"name": "text_editor_code_execution",
"input": {"command": "view", "path": "notes.txt"},
},
{
"type": "server_tool_use",
"id": "srvtoolu_03Bash",
"name": "bash_code_execution",
"input": {"command": "pwd"},
},
_WEATHER_TOOL_USE,
)
)
assert [(item.type, getattr(item, "call_id", item.id)) for item in response.output if item.type != "message"] == [
("code_interpreter_call", "srvtoolu_01Bash"),
("function_call", "toolu_01Weather"),
]
class TestHostedWebSearchReplay:
def test_emitted_hosted_search_output_round_trips_with_client_tool_result(self) -> None:
search_result: Final = {
@ -5063,6 +5216,75 @@ class TestHostedWebSearchReplay:
assert messages[2]["content"] == "Paris is sunny."
assert history == original
@pytest.mark.parametrize(
"leading_text", ((), ({"type": "text", "text": "Let me look that up."},)), ids=("alone", "after_text")
)
@pytest.mark.parametrize(
("server_tool_use", "expected_input"),
(
(
{"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_fetch", "input": {"url": _FETCHED_URL}},
{"url": _FETCHED_URL},
),
(
{"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_search", "input": {"query": "python"}},
{"query": "python"},
),
),
ids=("web_fetch", "web_search"),
)
def test_pending_claude_web_call_replays_to_anthropic_as_its_own_server_tool_use(
self,
leading_text: tuple[Mapping[str, object], ...],
server_tool_use: Mapping[str, object],
expected_input: Mapping[str, object],
) -> None:
response: Final = _anthropic_turn_on_responses((*leading_text, server_tool_use, _WEATHER_TOOL_USE))
history: Final = [
{"role": "user", "content": "Use the web and check the weather in Paris."},
*(item.model_dump(exclude_none=True) for item in response.output),
{"type": "function_call_output", "call_id": "toolu_01Weather", "output": "Paris is sunny."},
]
anthropic_turns: Final = anthropic_messages_pt(
messages=LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=history, responses_api_request={}
),
model="claude-sonnet-5-5",
llm_provider="anthropic",
)
assert [turn["role"] for turn in anthropic_turns] == ["user", "assistant", "user"]
assert [block for block in anthropic_turns[1]["content"] if block["type"] != "text"] == [
{"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": server_tool_use["name"], "input": expected_input},
{"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}},
]
assert [(block["type"], block["tool_use_id"]) for block in anthropic_turns[2]["content"]] == [
("tool_result", "toolu_01Weather")
]
@pytest.mark.parametrize(
("call_id", "status"),
(("ws_srvtoolu_01Fetch", "completed"), ("ws_68a1b2c3d4", "in_progress"), ("srvtoolu_01Fetch", "in_progress")),
ids=("completed-claude-call", "pending-non-claude-call", "pending-unprefixed-id"),
)
def test_web_call_that_is_not_a_pending_claude_call_replays_as_text_only(
self, call_id: str, status: Literal["completed", "in_progress"]
) -> None:
search: Final = ResponseFunctionWebSearch(
id=call_id, type="web_search_call", status=status, action=ActionOpenPage(type="open_page", url=_FETCHED_URL)
)
messages: Final = (
LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message(
input_item=search.model_dump(exclude_none=True)
)
)
assert messages == [
{"role": "assistant", "content": "Hosted web search: " + search.model_dump_json(exclude_none=True)}
]
@pytest.mark.parametrize(
"action",
(

View file

@ -356,6 +356,198 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to
)
_STREAMED_WEB_FETCH_CALL: Final[tuple[dict[str, object], ...]] = (
{
"type": "message_start",
"message": {
"id": "msg_01Fetch",
"type": "message",
"role": "assistant",
"content": [],
"usage": {"input_tokens": 10, "output_tokens": 1},
},
},
{
"type": "content_block_start",
"index": 0,
"content_block": {"type": "server_tool_use", "id": "srvtoolu_01Fetch", "name": "web_fetch"},
},
{
"type": "content_block_delta",
"index": 0,
"delta": {"type": "input_json_delta", "partial_json": '{"url": "https://example.com/"}'},
},
{"type": "content_block_stop", "index": 0},
)
_STREAMED_WEB_FETCH_RESULT: Final[tuple[dict[str, object], ...]] = (
{
"type": "content_block_start",
"index": 1,
"content_block": {
"type": "web_fetch_tool_result",
"tool_use_id": "srvtoolu_01Fetch",
"content": {"type": "web_fetch_result", "url": "https://example.com/"},
},
},
{"type": "content_block_stop", "index": 1},
)
def _streamed_text_then_weather_call(first_index: int) -> tuple[dict[str, object], ...]:
return (
{"type": "content_block_start", "index": first_index, "content_block": {"type": "text", "text": ""}},
{"type": "content_block_delta", "index": first_index, "delta": {"type": "text_delta", "text": "Fetched."}},
{"type": "content_block_stop", "index": first_index},
{
"type": "content_block_start",
"index": first_index + 1,
"content_block": {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {}},
},
{
"type": "content_block_delta",
"index": first_index + 1,
"delta": {"type": "input_json_delta", "partial_json": '{"city": "Paris"}'},
},
{"type": "content_block_stop", "index": first_index + 1},
{
"type": "message_delta",
"delta": {"stop_reason": "tool_use", "stop_sequence": None},
"usage": {"output_tokens": 20},
},
)
def _anthropic_web_fetch_stream_events(fetch_finished: bool) -> tuple[dict[str, object], ...]:
if fetch_finished:
return (*_STREAMED_WEB_FETCH_CALL, *_STREAMED_WEB_FETCH_RESULT, *_streamed_text_then_weather_call(2))
return (*_STREAMED_WEB_FETCH_CALL, *_streamed_text_then_weather_call(1))
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.parametrize(
("fetch_finished", "expected_status", "expected_search_events"),
[
pytest.param(
True,
"completed",
[
ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS,
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING,
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED,
ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
],
id="fetch_finished_in_this_turn",
),
pytest.param(
False,
"in_progress",
[
ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS,
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING,
ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
],
id="fetch_still_waiting_on_its_result",
),
],
)
@pytest.mark.asyncio
async def test_streamed_anthropic_web_fetch_is_a_hosted_open_page_call(
sync_mode: bool,
fetch_finished: bool,
expected_status: str,
expected_search_events: list[ResponsesAPIStreamEvents],
) -> None:
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
parser: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=sync_mode, json_mode=False)
chunks: Final = [parser.chunk_parser(event) for event in _anthropic_web_fetch_stream_events(fetch_finished)]
iterator: Final = LiteLLMCompletionStreamingIterator(
model="claude-sonnet-5-5",
litellm_custom_stream_wrapper=_FakeStreamWrapper(chunks),
request_input="Fetch example.com and check the weather in Paris.",
responses_api_request={
"tools": [
{"type": "web_fetch_20250910", "name": "web_fetch"},
{"type": "function", "name": "get_weather", "parameters": {"type": "object"}},
]
},
custom_llm_provider="anthropic",
)
events: Final = await _collect_events(iterator, sync_mode)
completed: Final = next(
event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
)
assert [
(item.id, item.status, item.action.model_dump())
for item in completed.response.output
if item.type == "web_search_call"
] == [("ws_srvtoolu_01Fetch", expected_status, {"type": "open_page", "url": "https://example.com/"})]
assert [
event.type
for event in events
if getattr(event, "item_id", getattr(getattr(event, "item", None), "id", None)) == "ws_srvtoolu_01Fetch"
] == expected_search_events
assert [item.name for item in completed.response.output if item.type == "function_call"] == ["get_weather"]
assert not any(
event.item.name == "web_fetch"
for event in events
if getattr(event, "type", None) == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED
and event.item.type == "function_call"
)
_STREAMED_PENDING_CODE_EXECUTION: Final[tuple[dict[str, object], ...]] = (
_STREAMED_WEB_FETCH_CALL[0],
{
"type": "content_block_start",
"index": 0,
"content_block": {"type": "server_tool_use", "id": "srvtoolu_01Bash", "name": "bash_code_execution"},
},
{
"type": "content_block_delta",
"index": 0,
"delta": {"type": "input_json_delta", "partial_json": '{"command": "ls"}'},
},
{"type": "content_block_stop", "index": 0},
*_streamed_text_then_weather_call(1),
)
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.asyncio
async def test_streamed_anthropic_code_execution_is_never_a_client_function_call(sync_mode: bool) -> None:
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
parser: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=sync_mode, json_mode=False)
iterator: Final = LiteLLMCompletionStreamingIterator(
model="claude-sonnet-5-5",
litellm_custom_stream_wrapper=_FakeStreamWrapper(
[parser.chunk_parser(event) for event in _STREAMED_PENDING_CODE_EXECUTION]
),
request_input="List the files, then check the weather in Paris.",
responses_api_request={
"tools": [
{"type": "code_execution_20250825", "name": "code_execution"},
{"type": "function", "name": "get_weather", "parameters": {"type": "object"}},
]
},
custom_llm_provider="anthropic",
)
events: Final = await _collect_events(iterator, sync_mode)
item_events: Final = [event for event in events if getattr(event, "item", None) is not None]
assert {event.item.call_id for event in item_events if event.item.type == "function_call"} == {"toolu_01Weather"}
completed: Final = next(
event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
)
assert [item.call_id for item in completed.response.output if item.type == "function_call"] == ["toolu_01Weather"]
def test_tool_calls_present_only_in_final_response_are_emitted_before_completed():
iterator = LiteLLMCompletionStreamingIterator(
model="test-model",

View file

@ -131,6 +131,52 @@ describe("ToolsSection utils", () => {
expect(result[1].callData).toBeUndefined();
});
it("should mark Claude server tools as called from provider_specific_fields.server_tool_calls", () => {
const log: Partial<LogEntry> = {
request_id: "test-server-tools",
proxy_server_request: {
tools: [
{ type: "web_search_20250305", name: "web_search", max_uses: 3 },
{ type: "function", function: { name: "get_weather", description: "Get weather" } },
{ type: "function", function: { name: "send_email", description: "Send email" } },
],
},
response: {
choices: [
{
message: {
tool_calls: [
{
id: "toolu_01Weather",
type: "function",
function: { name: "get_weather", arguments: '{"location": "Paris"}' },
},
],
provider_specific_fields: {
server_tool_calls: [
{
id: "srvtoolu_01Search",
type: "function",
function: { name: "web_search", arguments: '{"query": "paris weather"}' },
},
],
},
},
},
],
},
};
const result = parseToolsFromLog(log as LogEntry);
expect(result.map((tool) => [tool.name, tool.called, tool.callData?.id])).toEqual([
["web_search", true, "srvtoolu_01Search"],
["get_weather", true, "toolu_01Weather"],
["send_email", false, undefined],
]);
expect(result[0].callData?.arguments).toEqual({ query: "paris weather" });
});
it("should handle string format request and response", () => {
const log: Partial<LogEntry> = {
request_id: "test-5",

View file

@ -42,6 +42,24 @@ function extractToolsFromRequest(log: LogEntry): ToolDefinition[] {
return [];
}
interface ChatMessageToolCallFields {
tool_calls?: unknown;
provider_specific_fields?: { server_tool_calls?: unknown } | null;
}
function toolCallList(value: unknown): ToolCall[] {
return Array.isArray(value) ? value : [];
}
function extractChatMessageToolCalls(message: ChatMessageToolCallFields | undefined): ToolCall[] | null {
const serverToolCalls = message?.provider_specific_fields?.server_tool_calls;
const clientToolCalls = message?.tool_calls;
if (!Array.isArray(serverToolCalls) && !Array.isArray(clientToolCalls)) {
return null;
}
return [...toolCallList(serverToolCalls), ...toolCallList(clientToolCalls)];
}
/**
* Extract tool calls from response data
*/
@ -53,10 +71,9 @@ function extractToolCallsFromResponse(log: LogEntry): ToolCall[] {
// OpenAI format: response.choices[0].message.tool_calls
const choices = responseData.choices;
if (Array.isArray(choices) && choices.length > 0) {
const firstChoice = choices[0];
const message = firstChoice.message;
if (message && Array.isArray(message.tool_calls)) {
return message.tool_calls;
const chatToolCalls = extractChatMessageToolCalls(choices[0].message);
if (chatToolCalls) {
return chatToolCalls;
}
}