scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-12 08:42:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 455

fix(anthropic): let guardrails, tool usage, tracing, and Responses replay see Claude's server calls

Browse source 
Claude's server tool calls moved out of tool_calls into provider_specific_fields.server_tool_calls,
so every reader of tool_calls stopped seeing them. The chat guardrail handler now passes them to
output guardrails and writes rewrites back to server_tool_calls (stream and non-stream), tool usage
and registry discovery count them, otel, Datadog and the Logs Tools panel list them, and a pending
Claude web search or fetch item replayed to /v1/responses goes back to Anthropic as its own
server_tool_use instead of running again
This commit is contained in:
mateo-berri 2026-10-29 11:38:45 -07:00
parent 5e9fc38f05
commit 1a30066945
14 changed files with 643 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
litellm/integrations/datadog/datadog_llm_obs.py
View file

@ -44,6 +44,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
@ -315,7 +316,10 @@ def _to_dd_tool_calls(message: Mapping[str, object]) -> tuple[ToolCall, ...]:
tool_id=_text_field(tool_call, "id"),
type=_text_field(tool_call, "type", "function"),
)
for tool_call in (raw_tool_calls if isinstance(raw_tool_calls, list) else ())
for tool_call in (
*anthropic_server_tool_call_params(message),
*(raw_tool_calls if isinstance(raw_tool_calls, list) else ()),
)
if isinstance(tool_call, dict)
for function in [_mapping_field(tool_call, "function")]
)

11
litellm/integrations/otel/mappers/utils.py
View file

@ -12,6 +12,8 @@ from typing import Final
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue
from litellm.integrations.otel.model.payloads import LLMCallSpanData, ToolDefinition
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
@dataclass(frozen=True, slots=True)
@ -141,12 +143,13 @@ def output_messages(data: LLMCallSpanData) -> list:
def message_tool_calls(message: object) -> tuple[MessageToolCall, ...]:
tool_call_values: Final = (
*anthropic_server_tool_call_params(message),
*parts_of(message_field(message, "tool_calls")),
)
if not isinstance(message, dict):
return ()
tool_calls: Final = message.get("tool_calls")
if not isinstance(tool_calls, (list, tuple)) or not tool_calls:
return ()
return tuple(tool_call for value in tool_calls if (tool_call := _message_tool_call(value)) is not None)
return tuple(tool_call for value in tool_call_values if (tool_call := _message_tool_call(value)) is not None)
def _message_tool_call(value: object) -> MessageToolCall | None:

1
litellm/llms/anthropic/chat/handler.py
View file

@ -827,7 +827,6 @@ class ModelResponseIterator:
return text, tool_use, thinking_blocks, provider_specific_fields, reasoning_content
self.content_blocks.append(content_block)
if "partial_json" in content_block["delta"]:
# Only emit tool calls if we're in a client tool_use block, server_tool_use runs on Anthropic's side
# web_search_tool_result blocks also have input_json_delta but should not be treated as tool calls
# See: https://github.com/BerriAI/litellm/issues/17254
if self.current_content_block_type == "tool_use":

120
litellm/llms/openai/chat/guardrail_translation/handler.py
View file

@ -21,6 +21,7 @@ from collections.abc import Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Union, cast
from pydantic import TypeAdapter, ValidationError
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
@ -29,6 +30,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i
replayed_tool_search_tool_names,
)
from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
StreamingScanKey,
@ -50,7 +52,12 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
stream_item_items,
unappliable_request_rewrite,
)
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantToolCall,
ChatCompletionToolCallFunctionChunk,
ChatCompletionToolParam,
)
from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
coerce_stream_holdback_value,
)
@ -436,9 +443,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
text_task_mappings=text_task_mappings,
tool_call_task_mappings=tool_call_task_mappings,
)
server_tool_call_tasks: Final = tuple(_server_tool_call_tasks(response.choices))
tool_calls_with_server_calls: Final = [
*tool_calls_to_check,
*(_server_tool_call_input(call) for _, call in server_tool_call_tasks),
]
# Step 2: Apply guardrail to all texts and tool calls in batch
if texts_to_check or tool_calls_to_check:
if texts_to_check or tool_calls_with_server_calls:
# Use the real request_data if provided (proxy path), otherwise
# create a standalone dict (SDK / direct-call path).
if request_data is None:
@ -452,8 +464,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
if images_to_check:
inputs["images"] = images_to_check
if tool_calls_to_check:
inputs["tool_calls"] = tool_calls_to_check
if tool_calls_with_server_calls:
inputs["tool_calls"] = tool_calls_with_server_calls
# Include model information from the response if available
if hasattr(response, "model") and response.model:
inputs["model"] = response.model
@ -469,8 +481,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
returned_tool_calls: Final = guardrailed_inputs.get("tool_calls")
guardrailed_tool_calls: Final[list[dict[str, object]]] = (
cast(list[dict[str, object]], returned_tool_calls)
if isinstance(returned_tool_calls, list) and len(returned_tool_calls) == len(tool_calls_to_check)
else tool_calls_to_check
if isinstance(returned_tool_calls, list)
and len(returned_tool_calls) == len(tool_calls_with_server_calls)
else tool_calls_with_server_calls
)
# Step 3: Map guardrail responses back to original response structure
@ -488,6 +501,11 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
tool_calls=guardrailed_tool_calls,
task_mappings=tool_call_task_mappings,
)
self._apply_guardrail_responses_to_output_server_tool_calls(
response=response,
guardrailed_tool_calls=guardrailed_tool_calls[len(tool_calls_to_check) :],
task_mappings=tuple(mapping for mapping, _ in server_tool_call_tasks),
)
verbose_proxy_logger.debug("OpenAI Chat Completions: Processed output response: %s", response)
@ -681,6 +699,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
pre_guardrail_tool_calls=pre_guardrail_tool_calls,
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
)
self._write_ended_stream_server_tool_call_rewrites(
responses_so_far=responses_so_far,
guardrailed_response=model_response,
)
@staticmethod
def _rebuild_ended_stream_per_choice(
@ -939,6 +961,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if choice.message.tool_calls and isinstance(choice.message.tool_calls, list):
if len(choice.message.tool_calls) > 0:
return True
if anthropic_server_tool_call_params(choice.message):
return True
elif isinstance(response, ModelResponseStream):
for streaming_choice in response.choices:
if isinstance(streaming_choice, litellm.StreamingChoices):
@ -949,6 +973,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if streaming_choice.delta.tool_calls and isinstance(streaming_choice.delta.tool_calls, list):
if len(streaming_choice.delta.tool_calls) > 0:
return True
if anthropic_server_tool_call_params(streaming_choice.delta):
return True
return False
def _extract_output_text_images_and_tool_calls(
@ -1099,6 +1125,25 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if "name" in func_dict:
existing_tool_call.function.name = func_dict["name"]
@staticmethod
def _apply_guardrail_responses_to_output_server_tool_calls(
response: "ModelResponse",
guardrailed_tool_calls: Sequence[object],
task_mappings: Sequence[tuple[int, int]],
) -> None:
guardrailed_by_task: Final = MappingProxyType(dict(zip(task_mappings, guardrailed_tool_calls)))
for choice_idx, choice in enumerate(response.choices):
server_tool_calls = anthropic_server_tool_call_params(choice.message)
rewritten = tuple(
_rewritten_server_tool_call(server_tool_call, guardrailed_by_task.get((choice_idx, server_idx)))
for server_idx, server_tool_call in enumerate(server_tool_calls)
)
if rewritten != server_tool_calls:
choice.message.provider_specific_fields = {
**(choice.message.provider_specific_fields or {}),
"server_tool_calls": list(rewritten),
}
@staticmethod
def _string_choice_contents(response: "ModelResponse") -> tuple[str | None, ...]:
return tuple(
@ -1211,6 +1256,32 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
for fragment in tail:
fragment.function.arguments = ""
@staticmethod
def _write_ended_stream_server_tool_call_rewrites(
responses_so_far: Sequence["ModelResponseStream"],
guardrailed_response: "ModelResponse",
) -> None:
guardrailed_by_id: Final = MappingProxyType(
{
(guardrailed_response.choices[choice_position].index, server_tool_call["id"]): server_tool_call
for (choice_position, _), server_tool_call in _server_tool_call_tasks(guardrailed_response.choices)
}
)
if not guardrailed_by_id:
return
for response in responses_so_far:
for choice in response.choices:
streamed = anthropic_server_tool_call_params(choice.delta)
rewritten = tuple(
guardrailed_by_id.get((choice.index, server_tool_call["id"]), server_tool_call)
for server_tool_call in streamed
)
if rewritten != streamed:
choice.delta.provider_specific_fields = {
**(choice.delta.provider_specific_fields or {}),
"server_tool_calls": list(rewritten),
}
async def _apply_guardrail_responses_to_output_streaming(
self,
responses: list["ModelResponseStream"],
@ -1400,7 +1471,42 @@ def _stream_chunk_choices(item: object) -> Sequence[object]:
def _streamed_delta_tool_calls(delta: object) -> tuple[object, ...]:
function_call: Final = stream_item_field(delta, "function_call")
legacy: Final = () if function_call is None else (function_call,)
return stream_item_items(delta, "tool_calls") + legacy
return anthropic_server_tool_call_params(delta) + stream_item_items(delta, "tool_calls") + legacy
def _server_tool_call_tasks(
choices: Sequence[object],
) -> Iterator[tuple[tuple[int, int], ChatCompletionAssistantToolCall]]:
for choice_idx, choice in enumerate(choices):
if not isinstance(choice, Choices):
continue
for server_idx, server_tool_call in enumerate(anthropic_server_tool_call_params(choice.message)):
yield (choice_idx, server_idx), server_tool_call
def _server_tool_call_input(server_tool_call: ChatCompletionAssistantToolCall) -> dict[str, object]:
return {
"id": server_tool_call["id"],
"type": server_tool_call["type"],
"function": dict(server_tool_call["function"]),
}
_TOOL_CALL_FUNCTION_ADAPTER: Final = TypeAdapter(ChatCompletionToolCallFunctionChunk)
def _rewritten_server_tool_call(
server_tool_call: ChatCompletionAssistantToolCall, guardrailed: object
) -> ChatCompletionAssistantToolCall:
try:
function: Final = _TOOL_CALL_FUNCTION_ADAPTER.validate_python(stream_item_field(guardrailed, "function"))
except ValidationError:
return server_tool_call
return ChatCompletionAssistantToolCall(
id=server_tool_call["id"],
type=server_tool_call["type"],
function={**server_tool_call["function"], **function},
)
def _released_choices(responses_so_far: Sequence[object]) -> Iterator[object]:

17
litellm/proxy/db/spend_log_tool_index.py
View file

@ -63,11 +63,26 @@ def response_tool_call_names(completion_response: object) -> tuple[str, ...]:
)
return tuple(
stripped
for name in chain(called_names, _auto_executed_mcp_tool_names(completion_response))
for name in chain(
_server_tool_call_names(completion_response),
called_names,
_auto_executed_mcp_tool_names(completion_response),
)
if isinstance(name, str) and (stripped := name.strip())
)
def _server_tool_call_names(completion_response: object) -> tuple[str | None, ...]:
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
server_tool_calls: Final = chain.from_iterable(
anthropic_server_tool_call_params(message_field(choice, "message"))
for choice in parts_of(message_field(completion_response, "choices"))
)
return tuple(server_tool_call["function"].get("name") for server_tool_call in server_tool_calls)
class _ExecutedTool(BaseModel):
model_config = ConfigDict(frozen=True)

56
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -31,6 +31,7 @@ from openai.types.chat.chat_completion_named_tool_choice_param import (
from openai.types.responses import ResponseFunctionToolCall, ResponseFunctionWebSearch
from openai.types.responses.response_compaction_item import ResponseCompactionItem
from openai.types.responses.response_create_params import ResponseInputParam
from openai.types.responses.response_function_web_search import ActionOpenPage, ActionSearch
from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam
from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam
from openai.types.responses.tool_param import FunctionToolParam
@ -56,7 +57,12 @@ from litellm.litellm_core_utils.hidden_params import (
set_hidden_params,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.common_utils import anthropic_server_tool_calls, supports_anthropic_cache_control
from litellm.llms.anthropic.common_utils import (
anthropic_server_tool_call_params,
anthropic_server_tool_calls,
supports_anthropic_cache_control,
validated_object_mapping,
)
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support
from litellm.llms.openrouter.chat.transformation import openrouter_model_supports_cache_control
@ -66,11 +72,12 @@ from litellm.responses.litellm_completion_transformation.session_handler import
ResponsesSessionHandler,
StoredTurns,
)
from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex
from litellm.types.llms.anthropic import ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, AnthropicToolSearchToolRegex
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
ChatCompletionAssistantToolCall,
ChatCompletionAudioObject,
ChatCompletionCachedContent,
ChatCompletionFileObject,
@ -114,6 +121,7 @@ from litellm.types.responses.main import (
OutputImageGenerationCall,
OutputText,
OutputToolSearchCall,
web_search_call_tool_id,
)
from litellm.types.utils import (
ChatCompletionAnnotation,
@ -873,6 +881,29 @@ def _chain_session_metadata(chain_session: str, request_kwargs: Mapping[str, obj
return {key: {**_STR_KEY_DICT_ADAPTER.validate_python(metadata), CALLER_SESSION_ID_METADATA_KEY: chain_session}}
def _pending_claude_server_tool_call(search: ResponseFunctionWebSearch) -> ChatCompletionAssistantToolCall | None:
tool_id: Final = web_search_call_tool_id(search.id)
if search.status != "in_progress" or tool_id is None:
return None
if not tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX):
return None
match search.action:
case ActionSearch(query=str() as query):
return _server_tool_call(tool_id, "web_search", json.dumps({"query": query}))
case ActionOpenPage(url=str() as url):
return _server_tool_call(tool_id, "web_fetch", json.dumps({"url": url}))
case _:
return None
def _server_tool_call(tool_id: str, name: str, arguments: str) -> ChatCompletionAssistantToolCall:
return ChatCompletionAssistantToolCall(
id=tool_id,
type="function",
function=ChatCompletionToolCallFunctionChunk(name=name, arguments=arguments),
)
class LiteLLMCompletionResponsesConfig:
@staticmethod
def get_supported_openai_params(model: str) -> list:
@ -1778,10 +1809,22 @@ class LiteLLMCompletionResponsesConfig:
message_cache_control: Final = (
None if isinstance(content, list) else new_cache_control or previous_cache_control
)
new_server_tool_calls: Final = anthropic_server_tool_call_params(new_message)
server_tool_call_fields: Final = (
{
"provider_specific_fields": {
**(validated_object_mapping(last_message.get("provider_specific_fields")) or {}),
"server_tool_calls": [*anthropic_server_tool_call_params(last_message), *new_server_tool_calls],
}
}
if new_server_tool_calls
else {}
)
merged: Final = {
**{key: value for key, value in last_message.items() if key != "cache_control"},
"content": content,
**({} if message_cache_control is None else {"cache_control": message_cache_control}),
**server_tool_call_fields,
}
return cast( # cast-ok: preserves the assistant fields and content blocks
ChatCompletionAssistantMessage, merged
@ -2223,10 +2266,15 @@ class LiteLLMCompletionResponsesConfig:
content="Hosted web search: " + replay.model_dump_json(exclude_none=True),
)
]
hosted_search_text: Final = "Hosted web search: " + search.model_dump_json(exclude_none=True)
pending_server_tool_call: Final = _pending_claude_server_tool_call(search)
if pending_server_tool_call is None:
return [GenericChatCompletionMessage(role="assistant", content=hosted_search_text)]
return [
GenericChatCompletionMessage(
ChatCompletionResponseMessage(
role="assistant",
content="Hosted web search: " + search.model_dump_json(exclude_none=True),
content=hosted_search_text,
provider_specific_fields={"server_tool_calls": [pending_server_tool_call]},
)
]
if LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(input_item):

10
litellm/types/responses/main.py
View file

@ -92,6 +92,9 @@ def _is_tool_result_error(content: object, tool_name: str) -> bool:
return False
WEB_SEARCH_CALL_ID_PREFIX: Final = "ws_"
def build_web_search_call(
tool_id: str,
tool_input: object,
@ -101,13 +104,18 @@ def build_web_search_call(
) -> ResponseFunctionWebSearch:
content: Final = _result_content(result)
return ResponseFunctionWebSearch(
id=f"ws_{tool_id}",
id=f"{WEB_SEARCH_CALL_ID_PREFIX}{tool_id}",
type="web_search_call",
status=status or ("failed" if _is_tool_result_error(content, tool_name) else "completed"),
action=_web_fetch_action(tool_input) if tool_name == "web_fetch" else _web_search_action(tool_input, content),
)
def web_search_call_tool_id(call_id: str) -> str | None:
tool_id: Final = call_id.removeprefix(WEB_SEARCH_CALL_ID_PREFIX)
return None if tool_id == call_id else tool_id
class OutputImageGenerationCall(BaseLiteLLMOpenAIResponseObject):
"""An image generation call output"""

35
tests/unit/integrations/datadog/test_datadog_llm_obs.py
View file

@ -110,6 +110,41 @@ def test_output_tool_calls_use_the_datadog_tool_call_schema(logger: DataDogLLMOb
assert "function" not in message["tool_calls"][0]
def test_claude_server_tool_calls_are_reported_before_client_tool_calls(logger: DataDogLLMObsLogger) -> None:
payload = build(
logger,
response_message={
"role": "assistant",
"content": None,
"tool_calls": [ASSISTANT_TOOL_CALL],
"provider_specific_fields": {
"server_tool_calls": [
{
"id": "srvtoolu_01Search",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "paris weather"}'},
}
]
},
},
)
assert payload["meta"]["output"]["messages"][0]["tool_calls"] == [
{
"name": "web_search",
"arguments": {"query": "paris weather"},
"tool_id": "srvtoolu_01Search",
"type": "function",
},
{
"name": "get_weather",
"arguments": {"city": "Paris", "unit": "c"},
"tool_id": "call_abc123",
"type": "function",
},
]
def test_tool_call_identifiers_that_are_not_strings_are_blanked_not_stringified(logger: DataDogLLMObsLogger) -> None:
payload = build(
logger,

39
tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py
View file

@ -174,6 +174,45 @@ def test_openinference_output_tool_calls_preserve_calls_in_attributes_and_value(
]
def test_openinference_output_lists_claude_server_tool_calls_before_client_tool_calls():
data: Final = _llm_call(
choices_out=(
{
"finish_reason": "tool_calls",
"message": {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_paris",
"type": "function",
"function": {"name": "lookup_weather", "arguments": '{"city": "Paris"}'},
}
],
"provider_specific_fields": {
"server_tool_calls": [
{
"id": "srvtoolu_01Search",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "paris weather"}'},
}
]
},
},
},
)
)
attrs: Final = OpenInferenceMapper().map(data)
assert {key: value for key, value in attrs.items() if ".tool_calls." in key} == {
"llm.output_messages.0.message.tool_calls.0.tool_call.id": "srvtoolu_01Search",
"llm.output_messages.0.message.tool_calls.0.tool_call.function.name": "web_search",
"llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": '{"query": "paris weather"}',
"llm.output_messages.0.message.tool_calls.1.tool_call.id": "call_paris",
"llm.output_messages.0.message.tool_calls.1.tool_call.function.name": "lookup_weather",
"llm.output_messages.0.message.tool_calls.1.tool_call.function.arguments": '{"city": "Paris"}',
}
def test_openinference_responses_tool_calls_are_emitted_as_output_attributes():
data: Final = LLMCallSpanData.from_standard_logging_payload(
_responses_payload(

200
tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py
View file

@ -144,6 +144,42 @@ class MockMisalignedToolCallGuardrail(CustomGuardrail):
)
def _server_tool_call(call_id: str, name: str, arguments: Mapping[str, object]) -> dict[str, object]:
return {"id": call_id, "type": "function", "function": {"name": name, "arguments": json.dumps(arguments)}}
def _tool_call_ids_names_and_arguments(tool_calls) -> list[tuple[str, str, str]]:
return [
(tool_call["id"], tool_call["function"]["name"], tool_call["function"]["arguments"])
for tool_call in tool_calls
]
class QueryRedactingGuardrail(CustomGuardrail):
"""Returns copied tool calls with any ``query`` argument redacted, leaving other calls as they came."""
async def apply_guardrail(
self,
inputs: GenericGuardrailAPIInputs,
request_data: dict,
input_type: Literal["request", "response"],
logging_obj: Optional[Any] = None,
) -> GenericGuardrailAPIInputs:
def redacted(tool_call):
arguments = json.loads(tool_call["function"]["arguments"])
if "query" not in arguments:
return tool_call
return {
**tool_call,
"function": {**tool_call["function"], "arguments": json.dumps({**arguments, "query": "[REDACTED]"})},
}
return GenericGuardrailAPIInputs(
texts=inputs.get("texts", []),
tool_calls=[redacted(tool_call) for tool_call in inputs.get("tool_calls", [])],
)
class TestOpenAIChatCompletionsHandlerToolsInput:
"""Test input processing with tools (function definitions)"""
@ -906,6 +942,92 @@ class TestOpenAIChatCompletionsHandlerToolCallsOutput:
assert tool_calls[0].function.arguments == first_args
assert tool_calls[1].function.arguments == second_args
@pytest.mark.asyncio
async def test_output_guardrail_sees_a_response_that_only_carries_server_tool_calls(self):
handler = OpenAIChatCompletionsHandler()
guardrail = TestOpenAIChatCompletionsHandlerStreamingOutput._recording_guardrail()
response = ModelResponse(
model="claude-sonnet-4-5",
choices=[
Choices(
finish_reason="stop",
index=0,
message=Message(
content=None,
role="assistant",
provider_specific_fields={
"server_tool_calls": [_server_tool_call("srvtoolu_01Search", "web_search", {"query": "x"})]
},
),
)
],
)
await handler.process_output_response(response, guardrail)
assert _tool_call_ids_names_and_arguments(guardrail.seen_inputs[-1]["tool_calls"]) == [
("srvtoolu_01Search", "web_search", '{"query": "x"}')
]
@pytest.mark.asyncio
async def test_output_guardrail_rewrite_of_a_server_tool_call_lands_in_server_tool_calls_only(self):
handler = OpenAIChatCompletionsHandler()
weather_args = json.dumps({"city": "Paris", "query": "forecast"})
response = ModelResponse(
model="claude-sonnet-4-5",
choices=[
Choices(
finish_reason="tool_calls",
index=0,
message=Message(
content="Looking it up",
role="assistant",
tool_calls=[
ChatCompletionMessageToolCall(
id="call_1",
type="function",
function=Function(name="get_weather", arguments=json.dumps({"city": "Paris"})),
)
],
provider_specific_fields={
"server_tool_calls": [
_server_tool_call("srvtoolu_01Search", "web_search", {"query": "jane@example.com"})
]
},
),
),
Choices(
finish_reason="tool_calls",
index=1,
message=Message(
content=None,
role="assistant",
tool_calls=[
ChatCompletionMessageToolCall(
id="call_2",
type="function",
function=Function(name="get_weather", arguments=weather_args),
)
],
),
),
],
)
await handler.process_output_response(response, QueryRedactingGuardrail(guardrail_name="redact"))
first, second = response.choices
assert _tool_call_ids_names_and_arguments(first.message.provider_specific_fields["server_tool_calls"]) == [
("srvtoolu_01Search", "web_search", '{"query": "[REDACTED]"}')
]
assert [(call.id, call.function.arguments) for call in first.message.tool_calls] == [
("call_1", '{"city": "Paris"}')
]
assert [(call.id, call.function.arguments) for call in second.message.tool_calls] == [
("call_2", '{"city": "Paris", "query": "[REDACTED]"}')
]
assert not (second.message.provider_specific_fields or {}).get("server_tool_calls")
class MockPassThroughGuardrail(CustomGuardrail):
"""Mock guardrail that passes through without blocking - for testing streaming fallback behavior"""
@ -1462,6 +1584,61 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
assert chunks[0].choices[0].delta.content == "hello [MASKED]"
assert chunks[1].choices[0].delta.content in (None, "")
@staticmethod
def _ended_server_tool_call_stream_chunks() -> list:
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
def chunk(delta: Delta, finish_reason: Optional[str] = None) -> ModelResponseStream:
return ModelResponseStream(
id="chatcmpl-123",
created=1234567890,
model="claude-sonnet-4-5",
object="chat.completion.chunk",
choices=[StreamingChoices(index=0, delta=delta, finish_reason=finish_reason)],
)
search = {**_server_tool_call("srvtoolu_01Search", "web_search", {"query": "jane@example.com"}), "index": 0}
return [
chunk(Delta(content="Searching")),
chunk(Delta(content=None, provider_specific_fields={"server_tool_calls": [search]})),
chunk(Delta(content=None), finish_reason="stop"),
]
@pytest.mark.asyncio
async def test_ended_stream_guardrail_sees_streamed_server_tool_calls(self):
handler = OpenAIChatCompletionsHandler()
guardrail = self._recording_guardrail()
await handler.process_output_streaming_response(
responses_so_far=self._ended_server_tool_call_stream_chunks(),
guardrail_to_apply=guardrail,
litellm_logging_obj=None,
deliver_ended_stream_rewrites=True,
)
assert _tool_call_ids_names_and_arguments(guardrail.seen_inputs[-1]["tool_calls"]) == [
("srvtoolu_01Search", "web_search", '{"query": "jane@example.com"}')
]
@pytest.mark.asyncio
async def test_deliver_ended_stream_rewrites_writes_server_tool_call_arguments_back_into_chunks(self):
handler = OpenAIChatCompletionsHandler()
chunks = self._ended_server_tool_call_stream_chunks()
await handler.process_output_streaming_response(
responses_so_far=chunks,
guardrail_to_apply=QueryRedactingGuardrail(guardrail_name="redact"),
litellm_logging_obj=None,
deliver_ended_stream_rewrites=True,
)
delta = chunks[1].choices[0].delta
assert _tool_call_ids_names_and_arguments(delta.provider_specific_fields["server_tool_calls"]) == [
("srvtoolu_01Search", "web_search", '{"query": "[REDACTED]"}')
]
assert delta.tool_calls is None
assert chunks[0].choices[0].delta.content == "Searching"
class TestUndecoratedGuardrailIsRecorded:
"""LIT-5983 regression: the handler calls apply_guardrail bare, so a custom guardrail
@ -2323,6 +2500,29 @@ class TestStreamingScanKey:
assert len(ended_key.tool_calls) == 1 and "run_shell" in ended_key.tool_calls[0]
assert ended_key.tool_calls_in_flight is False
def test_server_tool_calls_hold_the_window_and_enter_the_key_once_the_stream_has_ended(self):
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
handler = OpenAIChatCompletionsHandler()
server_chunk = ModelResponseStream(
choices=[
StreamingChoices(
index=0,
delta=Delta(
content=None,
provider_specific_fields={
"server_tool_calls": [_server_tool_call("srvtoolu_01Search", "web_search", {"query": "x"})]
},
),
)
]
)
open_key = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk])
ended_key = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk, self._chunk(None, "stop")])
assert open_key.tool_calls_in_flight is True
assert len(ended_key.tool_calls) == 1 and "web_search" in ended_key.tool_calls[0]
assert ended_key != open_key
def test_text_after_the_first_choice_finishes_still_changes_the_key(self):
handler = OpenAIChatCompletionsHandler()
first_done = [self._chunk("a", index=0), self._chunk("b", finish_reason="stop", index=0)]

31
tests/unit/proxy/db/test_spend_log_tool_index.py
View file

@ -23,6 +23,7 @@ from litellm.proxy.db.spend_log_tool_index import (
)
from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import ModelResponse
from tests.unit.proxy.db.fake_prisma_engine import engine_call
@ -204,6 +205,36 @@ class TestResponseToolCallNames:
def test_blank_names_are_dropped(self):
assert response_tool_call_names(_response_with_tool_calls(" ", "real_tool")) == ("real_tool",)
@pytest.mark.parametrize("as_object", [False, True], ids=["dict", "model_response"])
def test_server_tool_calls_of_every_choice_are_named_before_client_tool_calls(self, as_object: bool):
def tool_call(call_id: str, name: str) -> dict[str, object]:
return {"id": call_id, "type": "function", "function": {"name": name, "arguments": "{}"}}
response: dict[str, object] = {
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": None,
"tool_calls": [tool_call("call_1", "get_weather")],
"provider_specific_fields": {"server_tool_calls": [tool_call("srvtoolu_01", "web_search")]},
},
},
{
"index": 1,
"message": {
"role": "assistant",
"content": "done",
"provider_specific_fields": {"server_tool_calls": [tool_call("srvtoolu_02", "web_fetch")]},
},
},
]
}
completion_response = ModelResponse(**response) if as_object else response
assert response_tool_call_names(completion_response) == ("web_search", "web_fetch", "get_weather")
def test_responses_api_output_function_calls(self):
# Regression: /v1/responses carries invocations in output[] items of
# type function_call, not in choices; they must reach the rollup.

69
tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py
View file

@ -5216,6 +5216,75 @@ class TestHostedWebSearchReplay:
assert messages[2]["content"] == "Paris is sunny."
assert history == original
@pytest.mark.parametrize(
"leading_text", ((), ({"type": "text", "text": "Let me look that up."},)), ids=("alone", "after_text")
)
@pytest.mark.parametrize(
("server_tool_use", "expected_input"),
(
(
{"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_fetch", "input": {"url": _FETCHED_URL}},
{"url": _FETCHED_URL},
),
(
{"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_search", "input": {"query": "python"}},
{"query": "python"},
),
),
ids=("web_fetch", "web_search"),
)
def test_pending_claude_web_call_replays_to_anthropic_as_its_own_server_tool_use(
self,
leading_text: tuple[Mapping[str, object], ...],
server_tool_use: Mapping[str, object],
expected_input: Mapping[str, object],
) -> None:
response: Final = _anthropic_turn_on_responses((*leading_text, server_tool_use, _WEATHER_TOOL_USE))
history: Final = [
{"role": "user", "content": "Use the web and check the weather in Paris."},
*(item.model_dump(exclude_none=True) for item in response.output),
{"type": "function_call_output", "call_id": "toolu_01Weather", "output": "Paris is sunny."},
]
anthropic_turns: Final = anthropic_messages_pt(
messages=LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=history, responses_api_request={}
),
model="claude-sonnet-5-5",
llm_provider="anthropic",
)
assert [turn["role"] for turn in anthropic_turns] == ["user", "assistant", "user"]
assert [block for block in anthropic_turns[1]["content"] if block["type"] != "text"] == [
{"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": server_tool_use["name"], "input": expected_input},
{"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}},
]
assert [(block["type"], block["tool_use_id"]) for block in anthropic_turns[2]["content"]] == [
("tool_result", "toolu_01Weather")
]
@pytest.mark.parametrize(
("call_id", "status"),
(("ws_srvtoolu_01Fetch", "completed"), ("ws_68a1b2c3d4", "in_progress"), ("srvtoolu_01Fetch", "in_progress")),
ids=("completed-claude-call", "pending-non-claude-call", "pending-unprefixed-id"),
)
def test_web_call_that_is_not_a_pending_claude_call_replays_as_text_only(
self, call_id: str, status: Literal["completed", "in_progress"]
) -> None:
search: Final = ResponseFunctionWebSearch(
id=call_id, type="web_search_call", status=status, action=ActionOpenPage(type="open_page", url=_FETCHED_URL)
)
messages: Final = (
LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message(
input_item=search.model_dump(exclude_none=True)
)
)
assert messages == [
{"role": "assistant", "content": "Hosted web search: " + search.model_dump_json(exclude_none=True)}
]
@pytest.mark.parametrize(
"action",
(

46
ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts
View file

@ -131,6 +131,52 @@ describe("ToolsSection utils", () => {
expect(result[1].callData).toBeUndefined();
});
it("should mark Claude server tools as called from provider_specific_fields.server_tool_calls", () => {
const log: Partial<LogEntry> = {
request_id: "test-server-tools",
proxy_server_request: {
tools: [
{ type: "web_search_20250305", name: "web_search", max_uses: 3 },
{ type: "function", function: { name: "get_weather", description: "Get weather" } },
{ type: "function", function: { name: "send_email", description: "Send email" } },
],
},
response: {
choices: [
{
message: {
tool_calls: [
{
id: "toolu_01Weather",
type: "function",
function: { name: "get_weather", arguments: '{"location": "Paris"}' },
},
],
provider_specific_fields: {
server_tool_calls: [
{
id: "srvtoolu_01Search",
type: "function",
function: { name: "web_search", arguments: '{"query": "paris weather"}' },
},
],
},
},
},
],
},
};
const result = parseToolsFromLog(log as LogEntry);
expect(result.map((tool) => [tool.name, tool.called, tool.callData?.id])).toEqual([
["web_search", true, "srvtoolu_01Search"],
["get_weather", true, "toolu_01Weather"],
["send_email", false, undefined],
]);
expect(result[0].callData?.arguments).toEqual({ query: "paris weather" });
});
it("should handle string format request and response", () => {
const log: Partial<LogEntry> = {
request_id: "test-5",

25
ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts
View file

@ -42,6 +42,24 @@ function extractToolsFromRequest(log: LogEntry): ToolDefinition[] {
return [];
}
interface ChatMessageToolCallFields {
tool_calls?: unknown;
provider_specific_fields?: { server_tool_calls?: unknown } | null;
}
function toolCallList(value: unknown): ToolCall[] {
return Array.isArray(value) ? value : [];
}
function extractChatMessageToolCalls(message: ChatMessageToolCallFields | undefined): ToolCall[] | null {
const serverToolCalls = message?.provider_specific_fields?.server_tool_calls;
const clientToolCalls = message?.tool_calls;
if (!Array.isArray(serverToolCalls) && !Array.isArray(clientToolCalls)) {
return null;
}
return [...toolCallList(serverToolCalls), ...toolCallList(clientToolCalls)];
}
/**
* Extract tool calls from response data
*/
@ -53,10 +71,9 @@ function extractToolCallsFromResponse(log: LogEntry): ToolCall[] {
// OpenAI format: response.choices[0].message.tool_calls
const choices = responseData.choices;
if (Array.isArray(choices) && choices.length > 0) {
const firstChoice = choices[0];
const message = firstChoice.message;
if (message && Array.isArray(message.tool_calls)) {
return message.tool_calls;
const chatToolCalls = extractChatMessageToolCalls(choices[0].message);
if (chatToolCalls) {
return chatToolCalls;
}
}