mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 1a30066945 into a1bd5feebe
This commit is contained in:
commit
f46c7645f0
27 changed files with 1672 additions and 147 deletions
|
|
@ -44,6 +44,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
|
|
@ -315,7 +316,10 @@ def _to_dd_tool_calls(message: Mapping[str, object]) -> tuple[ToolCall, ...]:
|
|||
tool_id=_text_field(tool_call, "id"),
|
||||
type=_text_field(tool_call, "type", "function"),
|
||||
)
|
||||
for tool_call in (raw_tool_calls if isinstance(raw_tool_calls, list) else ())
|
||||
for tool_call in (
|
||||
*anthropic_server_tool_call_params(message),
|
||||
*(raw_tool_calls if isinstance(raw_tool_calls, list) else ()),
|
||||
)
|
||||
if isinstance(tool_call, dict)
|
||||
for function in [_mapping_field(tool_call, "function")]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ from typing import Final
|
|||
|
||||
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue
|
||||
from litellm.integrations.otel.model.payloads import LLMCallSpanData, ToolDefinition
|
||||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
|
||||
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -141,12 +143,13 @@ def output_messages(data: LLMCallSpanData) -> list:
|
|||
|
||||
|
||||
def message_tool_calls(message: object) -> tuple[MessageToolCall, ...]:
|
||||
tool_call_values: Final = (
|
||||
*anthropic_server_tool_call_params(message),
|
||||
*parts_of(message_field(message, "tool_calls")),
|
||||
)
|
||||
if not isinstance(message, dict):
|
||||
return ()
|
||||
tool_calls: Final = message.get("tool_calls")
|
||||
if not isinstance(tool_calls, (list, tuple)) or not tool_calls:
|
||||
return ()
|
||||
return tuple(tool_call for value in tool_calls if (tool_call := _message_tool_call(value)) is not None)
|
||||
return tuple(tool_call for value in tool_call_values if (tool_call := _message_tool_call(value)) is not None)
|
||||
|
||||
|
||||
def _message_tool_call(value: object) -> MessageToolCall | None:
|
||||
|
|
|
|||
|
|
@ -2004,12 +2004,39 @@ def _tool_call_ids_with_a_kept_server_result(message: object) -> frozenset[str]:
|
|||
return frozenset()
|
||||
return frozenset(
|
||||
tool_call_id
|
||||
for tool_call in parts_of(message_field(message, "tool_calls"))
|
||||
for tool_call in (
|
||||
*parts_of(message_field(message, "tool_calls")),
|
||||
*parts_of(message_field(provider_specific_fields, "server_tool_calls")),
|
||||
)
|
||||
if isinstance(tool_call_id := message_field(tool_call, "id"), str)
|
||||
and tool_call_is_rebuilt_as_server_tool_use(tool_call_id, provider_specific_fields)
|
||||
)
|
||||
|
||||
|
||||
def _assistant_tool_calls_with_server_tool_calls(
|
||||
message: ChatCompletionAssistantMessage,
|
||||
) -> Sequence[ChatCompletionAssistantToolCall] | None:
|
||||
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
|
||||
|
||||
tool_calls: Final = message.get("tool_calls")
|
||||
server_tool_calls: Final = anthropic_server_tool_call_params(message)
|
||||
if not server_tool_calls:
|
||||
return tool_calls
|
||||
client_tool_call_ids: Final = frozenset(tool_call["id"] for tool_call in tool_calls or ())
|
||||
return (
|
||||
*(tool_call for tool_call in server_tool_calls if tool_call["id"] not in client_tool_call_ids),
|
||||
*(tool_calls or ()),
|
||||
)
|
||||
|
||||
|
||||
def _server_tool_call_ids(message: object) -> frozenset[str]:
|
||||
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
|
||||
|
||||
return frozenset(
|
||||
tool_call["id"] for tool_call in anthropic_server_tool_call_params(message) if tool_call["id"] is not None
|
||||
)
|
||||
|
||||
|
||||
def _server_tool_answers_replaced_after(messages: Sequence[object], index: int) -> frozenset[str]:
|
||||
kept_server_result_ids: Final = _tool_call_ids_with_a_kept_server_result(messages[index])
|
||||
if not kept_server_result_ids:
|
||||
|
|
@ -2069,10 +2096,11 @@ _AnthropicToolInvokeItem: TypeAlias = AnthropicMessagesToolUseParam | dict[str,
|
|||
|
||||
|
||||
def convert_to_anthropic_tool_invoke(
|
||||
tool_calls: list[ChatCompletionAssistantToolCall],
|
||||
tool_calls: Sequence[ChatCompletionAssistantToolCall],
|
||||
web_search_results: Sequence[object] | None = None,
|
||||
tool_results: Sequence[object] | None = None,
|
||||
answered_tool_ids: Container[str] = frozenset(),
|
||||
server_tool_call_ids: Container[str] = frozenset(),
|
||||
) -> list[_AnthropicToolInvokeItem]:
|
||||
"""
|
||||
OpenAI tool invokes:
|
||||
|
|
@ -2150,6 +2178,10 @@ def convert_to_anthropic_tool_invoke(
|
|||
anthropic_tool_invoke.append(server_tool_result)
|
||||
elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids:
|
||||
continue
|
||||
elif tool_id in server_tool_call_ids and tool_id not in answered_tool_ids:
|
||||
anthropic_tool_invoke.append(
|
||||
{"type": "server_tool_use", "id": tool_id, "name": tool_name, "input": tool_input}
|
||||
)
|
||||
else:
|
||||
sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id)
|
||||
_anthropic_tool_use_param = AnthropicMessagesToolUseParam(
|
||||
|
|
@ -3109,7 +3141,7 @@ def anthropic_messages_pt(
|
|||
# If so, we need to interleave thinking blocks with tool call groups
|
||||
# to preserve the original content block ordering.
|
||||
# Fixes: https://github.com/BerriAI/litellm/issues/23047
|
||||
assistant_tool_calls = assistant_content_block.get("tool_calls")
|
||||
assistant_tool_calls = _assistant_tool_calls_with_server_tool_calls(assistant_content_block)
|
||||
_has_server_tool_calls = False
|
||||
if assistant_tool_calls is not None:
|
||||
for _tc in assistant_tool_calls:
|
||||
|
|
@ -3142,6 +3174,7 @@ def anthropic_messages_pt(
|
|||
web_search_results=_web_search_results_tc,
|
||||
tool_results=_tool_results_tc,
|
||||
answered_tool_ids=client_answered_tool_ids,
|
||||
server_tool_call_ids=_server_tool_call_ids(assistant_content_block),
|
||||
)
|
||||
|
||||
# Group tool invoke results into (server_tool_use, result) pairs
|
||||
|
|
@ -3307,6 +3340,7 @@ def anthropic_messages_pt(
|
|||
web_search_results=_web_search_results,
|
||||
tool_results=_tool_results,
|
||||
answered_tool_ids=client_answered_tool_ids,
|
||||
server_tool_call_ids=_server_tool_call_ids(assistant_content_block),
|
||||
)
|
||||
|
||||
# Prevent "tool_use ids must be unique" errors by filtering duplicates
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from typing_extensions import ReadOnly, TypedDict, assert_never
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.anthropic.common_utils import anthropic_server_tool_calls
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import (
|
||||
OPENAI_MAX_TOOL_NAME_LENGTH,
|
||||
LiteLLMAnthropicMessagesAdapter,
|
||||
|
|
@ -1317,9 +1318,9 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
if built_response is not None and hasattr(built_response, "choices") and built_response.choices:
|
||||
model_response: Final = cast(ModelResponse, built_response)
|
||||
first_choice: Final = cast(Choices, model_response.choices[0])
|
||||
tool_calls_list: Final = cast(
|
||||
list[ChatCompletionMessageToolCall] | None,
|
||||
first_choice.message.tool_calls,
|
||||
tool_calls_list: Final = (
|
||||
*anthropic_server_tool_calls(first_choice.message),
|
||||
*cast(list[ChatCompletionMessageToolCall], first_choice.message.tool_calls or []),
|
||||
)
|
||||
string_so_far = first_choice.message.content
|
||||
pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ())
|
||||
|
|
@ -1327,7 +1328,7 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
if string_so_far:
|
||||
guardrail_inputs["texts"] = [string_so_far]
|
||||
if tool_calls_list:
|
||||
guardrail_inputs["tool_calls"] = tool_calls_list
|
||||
guardrail_inputs["tool_calls"] = list(tool_calls_list)
|
||||
|
||||
try:
|
||||
prepared_request_data = self._prepare_request_data(
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ from litellm.types.llms.openai import (
|
|||
ChatCompletionToolCallFunctionChunk,
|
||||
)
|
||||
from litellm.types.responses.main import (
|
||||
HOSTED_WEB_TOOL_NAMES,
|
||||
OutputCodeInterpreterCall,
|
||||
build_code_interpreter_log_outputs,
|
||||
build_web_search_call,
|
||||
|
|
@ -754,6 +755,8 @@ class ModelResponseIterator:
|
|||
self._server_tool_inputs: dict[str, object] = {}
|
||||
self.tool_results: list[dict[str, Any]] = []
|
||||
self._current_server_tool_id: str | None = None
|
||||
self._current_server_tool_name: str = ""
|
||||
self._server_tool_calls: tuple[ChatCompletionToolCallChunk, ...] = ()
|
||||
self._container_id: str | None = None
|
||||
|
||||
@property
|
||||
|
|
@ -824,10 +827,9 @@ class ModelResponseIterator:
|
|||
return text, tool_use, thinking_blocks, provider_specific_fields, reasoning_content
|
||||
self.content_blocks.append(content_block)
|
||||
if "partial_json" in content_block["delta"]:
|
||||
# Only emit tool calls if we're in a tool_use or server_tool_use block
|
||||
# web_search_tool_result blocks also have input_json_delta but should not be treated as tool calls
|
||||
# See: https://github.com/BerriAI/litellm/issues/17254
|
||||
if self.current_content_block_type in ("tool_use", "server_tool_use"):
|
||||
if self.current_content_block_type == "tool_use":
|
||||
tool_use = cast(
|
||||
ChatCompletionToolCallChunk,
|
||||
{
|
||||
|
|
@ -926,7 +928,7 @@ class ModelResponseIterator:
|
|||
def _web_search_call_snapshot(self) -> dict[str, object]:
|
||||
return dict(self._web_search_calls)
|
||||
|
||||
def _complete_web_search_call(self, result: dict[str, object]) -> None:
|
||||
def _complete_web_search_call(self, result: dict[str, object], tool_name: str) -> None:
|
||||
tool_use_id: Final = result.get("tool_use_id")
|
||||
if not isinstance(tool_use_id, str) or tool_use_id not in self._web_search_calls:
|
||||
return
|
||||
|
|
@ -934,6 +936,7 @@ class ModelResponseIterator:
|
|||
tool_id=tool_use_id,
|
||||
tool_input=self._server_tool_inputs.get(tool_use_id, {}),
|
||||
result=result,
|
||||
tool_name=tool_name,
|
||||
)
|
||||
|
||||
def _build_code_interpreter_results(self) -> list:
|
||||
|
|
@ -1013,7 +1016,6 @@ class ModelResponseIterator:
|
|||
content_block_start["content_block"]["type"] == "tool_use"
|
||||
or content_block_start["content_block"]["type"] == "server_tool_use"
|
||||
):
|
||||
self.tool_index += 1
|
||||
# Reverse-map the (sanitized) tool name back to the
|
||||
# caller's original. No-op when the map is empty.
|
||||
_stream_tool_name = content_block_start["content_block"]["name"]
|
||||
|
|
@ -1022,35 +1024,39 @@ class ModelResponseIterator:
|
|||
# Use empty string for arguments in content_block_start - actual arguments
|
||||
# come in subsequent content_block_delta chunks and get accumulated.
|
||||
# Using str(input) here would prepend '{}' causing invalid JSON accumulation.
|
||||
tool_use = ChatCompletionToolCallChunk(
|
||||
id=content_block_start["content_block"]["id"],
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=_stream_tool_name,
|
||||
arguments="",
|
||||
),
|
||||
index=self.tool_index,
|
||||
)
|
||||
if content_block_start["content_block"]["type"] == "tool_use":
|
||||
self.tool_index += 1
|
||||
tool_use = ChatCompletionToolCallChunk(
|
||||
id=content_block_start["content_block"]["id"],
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=_stream_tool_name,
|
||||
arguments="",
|
||||
),
|
||||
index=self.tool_index,
|
||||
)
|
||||
# Track server tool use inputs for code_interpreter_results.
|
||||
# The initial input in content_block_start is typically {}
|
||||
# for streaming; the full input arrives via input_json_delta
|
||||
# and is assembled at content_block_stop.
|
||||
if content_block_start["content_block"]["type"] == "server_tool_use":
|
||||
self._current_server_tool_id = content_block_start["content_block"]["id"]
|
||||
self._current_server_tool_name = _stream_tool_name
|
||||
tool_input: Final = content_block_start["content_block"].get("input", {})
|
||||
self._server_tool_inputs[self._current_server_tool_id] = tool_input
|
||||
if _stream_tool_name == "web_search":
|
||||
if _stream_tool_name in HOSTED_WEB_TOOL_NAMES:
|
||||
self._web_search_calls[self._current_server_tool_id] = build_web_search_call(
|
||||
self._current_server_tool_id,
|
||||
tool_input,
|
||||
{"content": []},
|
||||
status="in_progress",
|
||||
tool_name=_stream_tool_name,
|
||||
)
|
||||
provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
|
||||
# Include caller information if present (for programmatic tool calling)
|
||||
if "caller" in content_block_start["content_block"]:
|
||||
caller_data: Final = content_block_start["content_block"]["caller"]
|
||||
if caller_data:
|
||||
if caller_data and tool_use is not None:
|
||||
tool_use["caller"] = cast(dict[str, Any], caller_data)
|
||||
elif content_block_start["content_block"]["type"] == "redacted_thinking":
|
||||
(
|
||||
|
|
@ -1081,7 +1087,7 @@ class ModelResponseIterator:
|
|||
# The full content comes in content_block_start, not in deltas
|
||||
# See: https://github.com/BerriAI/litellm/issues/17737
|
||||
self.web_search_results.append(content_block_start["content_block"])
|
||||
self._complete_web_search_call(content_block_start["content_block"])
|
||||
self._complete_web_search_call(content_block_start["content_block"], "web_search")
|
||||
provider_specific_fields["web_search_results"] = self.web_search_results
|
||||
provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
|
||||
elif content_type == "web_fetch_tool_result":
|
||||
|
|
@ -1089,7 +1095,9 @@ class ModelResponseIterator:
|
|||
# The full content comes in content_block_start, not in deltas
|
||||
# Fixes: https://github.com/BerriAI/litellm/issues/18137
|
||||
self.web_search_results.append(content_block_start["content_block"])
|
||||
self._complete_web_search_call(content_block_start["content_block"], "web_fetch")
|
||||
provider_specific_fields["web_search_results"] = self.web_search_results
|
||||
provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
|
||||
else:
|
||||
self.tool_results.append(content_block_start["content_block"])
|
||||
provider_specific_fields["tool_results"] = self.tool_results
|
||||
|
|
@ -1104,7 +1112,7 @@ class ModelResponseIterator:
|
|||
# check if tool call content block - only for tool_use and server_tool_use blocks
|
||||
if self.current_content_block_type in ("tool_use", "server_tool_use"):
|
||||
is_empty: Final = self.check_empty_tool_call_args()
|
||||
if is_empty:
|
||||
if is_empty and self.current_content_block_type == "tool_use":
|
||||
tool_use = ChatCompletionToolCallChunk(
|
||||
id=None,
|
||||
type="function",
|
||||
|
|
@ -1128,6 +1136,29 @@ class ModelResponseIterator:
|
|||
self._server_tool_inputs[self._current_server_tool_id] = json.loads(args)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
server_tool_input: Final = self._server_tool_inputs.get(self._current_server_tool_id, {})
|
||||
if self._current_server_tool_id in self._web_search_calls:
|
||||
self._web_search_calls[self._current_server_tool_id] = build_web_search_call(
|
||||
self._current_server_tool_id,
|
||||
server_tool_input,
|
||||
{"content": []},
|
||||
status="in_progress",
|
||||
tool_name=self._current_server_tool_name,
|
||||
)
|
||||
provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
|
||||
self._server_tool_calls = (
|
||||
*self._server_tool_calls,
|
||||
ChatCompletionToolCallChunk(
|
||||
id=self._current_server_tool_id,
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=self._current_server_tool_name,
|
||||
arguments=json.dumps(server_tool_input),
|
||||
),
|
||||
index=len(self._server_tool_calls),
|
||||
),
|
||||
)
|
||||
provider_specific_fields["server_tool_calls"] = list(self._server_tool_calls)
|
||||
self._current_server_tool_id = None
|
||||
if self.is_response_format_tool:
|
||||
text = self._flush_response_format_buffer()
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from types import MappingProxyType
|
|||
from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, cast
|
||||
|
||||
import httpx
|
||||
from openai.types.responses.response_function_web_search import ResponseFunctionWebSearch
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
|
|
@ -62,6 +63,7 @@ from litellm.types.llms.anthropic import (
|
|||
ANTHROPIC_ADVISOR_TOOL_TYPE,
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_HOSTED_TOOLS,
|
||||
ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX,
|
||||
ANTHROPIC_TOOL_SEARCH_TOOL_TYPES,
|
||||
AllAnthropicPassThroughMessageValues,
|
||||
AllAnthropicToolsValues,
|
||||
|
|
@ -94,6 +96,7 @@ from litellm.types.llms.openai import (
|
|||
OpenAIWebSearchOptions,
|
||||
)
|
||||
from litellm.types.responses.main import (
|
||||
HOSTED_WEB_TOOL_NAMES,
|
||||
OutputCodeInterpreterCall,
|
||||
build_code_interpreter_log_outputs,
|
||||
build_web_search_call,
|
||||
|
|
@ -251,6 +254,29 @@ def _validated_defer_loading(value: object) -> bool:
|
|||
raise ValueError("defer_loading must be a boolean") from exc
|
||||
|
||||
|
||||
def _hosted_web_tool_use(block: object) -> tuple[str, str, object] | None:
|
||||
match block:
|
||||
case {
|
||||
"type": "server_tool_use",
|
||||
"name": str() as name,
|
||||
"id": str() as call_id,
|
||||
"input": object() as tool_input,
|
||||
} if name in HOSTED_WEB_TOOL_NAMES:
|
||||
return call_id, name, tool_input
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _hosted_web_call(call_id: str, tool_name: str, tool_input: object, result: object) -> ResponseFunctionWebSearch:
|
||||
return build_web_search_call(
|
||||
tool_id=call_id,
|
||||
tool_input=tool_input,
|
||||
result=result,
|
||||
status=None if result is not None else "in_progress",
|
||||
tool_name=tool_name,
|
||||
)
|
||||
|
||||
|
||||
class _AnthropicUsageIteration(TypedDict, total=False):
|
||||
"""One entry of the ``usage.iterations`` array on an Anthropic response."""
|
||||
|
||||
|
|
@ -2850,6 +2876,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
)
|
||||
return usage
|
||||
|
||||
@staticmethod
|
||||
def _is_server_tool_call(tool_call: ChatCompletionToolCallChunk) -> bool:
|
||||
tool_call_id: Final = tool_call["id"]
|
||||
return tool_call_id is not None and tool_call_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
|
||||
|
||||
def _build_code_by_id_map(self, tool_calls: list[ChatCompletionToolCallChunk]) -> dict[str, str]:
|
||||
code_by_id: Final[dict[str, str]] = {}
|
||||
for tc in tool_calls:
|
||||
|
|
@ -2894,29 +2925,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
self,
|
||||
web_search_results: Sequence[object],
|
||||
completion_response: Mapping[str, object],
|
||||
) -> list[object]:
|
||||
) -> list[ResponseFunctionWebSearch]:
|
||||
content: Final = completion_response.get("content")
|
||||
blocks: Final = content if isinstance(content, Sequence) else ()
|
||||
inputs: Final = {
|
||||
call_id: tool_input
|
||||
for block in blocks
|
||||
if isinstance(block, Mapping)
|
||||
and block.get("type") == "server_tool_use"
|
||||
and block.get("name") == "web_search"
|
||||
and isinstance((call_id := block.get("id")), str)
|
||||
and isinstance((tool_input := block.get("input")), Mapping)
|
||||
results_by_call: Final = {
|
||||
(tool_use_id, result.get("type")): result
|
||||
for result in web_search_results
|
||||
if isinstance(result, dict) and isinstance((tool_use_id := result.get("tool_use_id")), str)
|
||||
}
|
||||
return [
|
||||
build_web_search_call(
|
||||
tool_id=tool_use_id,
|
||||
tool_input=inputs.get(tool_use_id, {}),
|
||||
result=result,
|
||||
)
|
||||
for result in web_search_results
|
||||
if isinstance(result, dict)
|
||||
and result.get("type") == "web_search_tool_result"
|
||||
and isinstance((tool_use_id := result.get("tool_use_id")), str)
|
||||
and tool_use_id in inputs
|
||||
_hosted_web_call(call_id, tool_name, tool_input, results_by_call.get((call_id, f"{tool_name}_tool_result")))
|
||||
for call_id, tool_name, tool_input in filter(None, map(_hosted_web_tool_use, blocks))
|
||||
]
|
||||
|
||||
def _build_provider_specific_fields(
|
||||
|
|
@ -2940,10 +2959,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
|
||||
if web_search_results is not None:
|
||||
provider_specific_fields["web_search_results"] = web_search_results
|
||||
provider_specific_fields["web_search_calls"] = self._build_web_search_calls(
|
||||
web_search_results,
|
||||
completion_response,
|
||||
)
|
||||
|
||||
web_search_calls: Final = self._build_web_search_calls(web_search_results or (), completion_response)
|
||||
if web_search_calls:
|
||||
provider_specific_fields["web_search_calls"] = web_search_calls
|
||||
|
||||
if tool_results is not None:
|
||||
provider_specific_fields["tool_results"] = tool_results
|
||||
|
|
@ -2965,6 +2984,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
if compaction_blocks is not None:
|
||||
provider_specific_fields["compaction_blocks"] = compaction_blocks
|
||||
|
||||
server_tool_calls: Final = [tc for tc in tool_calls if self._is_server_tool_call(tc)]
|
||||
if server_tool_calls:
|
||||
provider_specific_fields["server_tool_calls"] = server_tool_calls
|
||||
|
||||
refusal_fields: Final = refusal_provider_specific_fields(
|
||||
REFUSAL_STOP_FIELDS.validate_python(completion_response)
|
||||
)
|
||||
|
|
@ -3024,6 +3047,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
):
|
||||
text_content = prefix_prompt + text_content
|
||||
|
||||
client_tool_calls: Final = [tc for tc in tool_calls if not self._is_server_tool_call(tc)]
|
||||
provider_specific_fields: Final = self._build_provider_specific_fields(
|
||||
completion_response,
|
||||
citations,
|
||||
|
|
@ -3036,7 +3060,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
|
||||
json_mode_message, tool_calls_for_message, json_extra_content = self._resolve_json_mode_non_streaming(
|
||||
json_mode=json_mode,
|
||||
tool_calls=tool_calls,
|
||||
tool_calls=client_tool_calls,
|
||||
response_format_schema=response_format_schema,
|
||||
)
|
||||
merged_text = text_content or ""
|
||||
|
|
|
|||
|
|
@ -60,10 +60,10 @@ from litellm.types.llms.anthropic import (
|
|||
AnthropicThinkingParam,
|
||||
)
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantToolCall
|
||||
from litellm.types.proxy.auth.special_headers import SpecialHeaders
|
||||
from litellm.types.proxy.model_listing import ModelInfoResponse
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.types.utils import ChatCompletionMessageToolCall, LlmProviders
|
||||
|
||||
_MessageT = TypeVar("_MessageT")
|
||||
|
||||
|
|
@ -116,6 +116,7 @@ _DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)")
|
|||
_CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
|
||||
OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
|
||||
OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
|
||||
_SERVER_TOOL_CALLS_ADAPTER: Final = TypeAdapter(tuple[ChatCompletionAssistantToolCall, ...])
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
|
|
@ -2074,6 +2075,21 @@ def _replayed_server_tool_use(block: object) -> _ReplayedServerToolUse | None:
|
|||
return None
|
||||
|
||||
|
||||
def anthropic_server_tool_call_params(message: object) -> tuple[ChatCompletionAssistantToolCall, ...]:
|
||||
fields: Final = validated_object_mapping(message_field(message, "provider_specific_fields"))
|
||||
try:
|
||||
return _SERVER_TOOL_CALLS_ADAPTER.validate_python(fields.get("server_tool_calls", ()) if fields else ())
|
||||
except ValidationError:
|
||||
return ()
|
||||
|
||||
|
||||
def anthropic_server_tool_calls(message: object) -> tuple[ChatCompletionMessageToolCall, ...]:
|
||||
return tuple(
|
||||
ChatCompletionMessageToolCall(id=call["id"], type=call["type"], function=dict(call["function"]))
|
||||
for call in anthropic_server_tool_call_params(message)
|
||||
)
|
||||
|
||||
|
||||
def tool_call_is_rebuilt_as_server_tool_use(
|
||||
tool_call_id: object, provider_specific_fields: object, answered_tool_ids: Container[str] = frozenset()
|
||||
) -> bool:
|
||||
|
|
|
|||
|
|
@ -45,6 +45,25 @@ def _thought_signature(provider_specific_fields: object) -> str | None:
|
|||
return signature if isinstance(signature, str) else None
|
||||
|
||||
|
||||
def _object_sequence(value: object) -> Sequence[object] | None:
|
||||
return value if isinstance(value, (list, tuple)) else None
|
||||
|
||||
|
||||
def _server_tool_blocks(message: object) -> tuple[Mapping[str, object], ...]:
|
||||
server_tool_calls: Final = anthropic_server_tool_call_params(message)
|
||||
if not server_tool_calls:
|
||||
return ()
|
||||
fields: Final = _as_string_mapping(_optional_attr(message, "provider_specific_fields")) or {}
|
||||
return tuple(
|
||||
convert_to_anthropic_tool_invoke(
|
||||
server_tool_calls,
|
||||
web_search_results=_object_sequence(fields.get("web_search_results")),
|
||||
tool_results=_object_sequence(fields.get("tool_results")),
|
||||
server_tool_call_ids=frozenset(tool_call["id"] for tool_call in server_tool_calls),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _compaction_blocks(provider_specific_fields: object) -> tuple[Mapping[str, object], ...]:
|
||||
fields: Final = _as_string_mapping(provider_specific_fields)
|
||||
raw_blocks: Final = fields.get("compaction_blocks") if fields is not None else None
|
||||
|
|
@ -130,11 +149,13 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
THOUGHT_SIGNATURE_SEPARATOR,
|
||||
convert_to_anthropic_tool_invoke,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import (
|
||||
reasoning_effort_from_thinking,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
anthropic_server_tool_call_params,
|
||||
eager_input_streaming_flag,
|
||||
is_empty_unsigned_thinking_block,
|
||||
normalize_anthropic_tool_use_id,
|
||||
|
|
@ -1428,6 +1449,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
).model_dump()
|
||||
)
|
||||
|
||||
new_content.extend(dict(block) for block in _server_tool_blocks(choice.message))
|
||||
# Handle text content
|
||||
if choice.message.content is not None and (choice.message.content != "" or not compaction_blocks):
|
||||
new_content.append(
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from collections.abc import Iterator, Mapping, Sequence
|
|||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Union, cast
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -29,6 +30,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i
|
|||
replayed_tool_search_tool_names,
|
||||
)
|
||||
from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder
|
||||
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import (
|
||||
BaseTranslation,
|
||||
StreamingScanKey,
|
||||
|
|
@ -50,7 +52,12 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
|
|||
stream_item_items,
|
||||
unappliable_request_rewrite,
|
||||
)
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionAssistantToolCall,
|
||||
ChatCompletionToolCallFunctionChunk,
|
||||
ChatCompletionToolParam,
|
||||
)
|
||||
from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
|
||||
coerce_stream_holdback_value,
|
||||
)
|
||||
|
|
@ -436,9 +443,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
text_task_mappings=text_task_mappings,
|
||||
tool_call_task_mappings=tool_call_task_mappings,
|
||||
)
|
||||
server_tool_call_tasks: Final = tuple(_server_tool_call_tasks(response.choices))
|
||||
tool_calls_with_server_calls: Final = [
|
||||
*tool_calls_to_check,
|
||||
*(_server_tool_call_input(call) for _, call in server_tool_call_tasks),
|
||||
]
|
||||
|
||||
# Step 2: Apply guardrail to all texts and tool calls in batch
|
||||
if texts_to_check or tool_calls_to_check:
|
||||
if texts_to_check or tool_calls_with_server_calls:
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
|
|
@ -452,8 +464,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
if images_to_check:
|
||||
inputs["images"] = images_to_check
|
||||
if tool_calls_to_check:
|
||||
inputs["tool_calls"] = tool_calls_to_check
|
||||
if tool_calls_with_server_calls:
|
||||
inputs["tool_calls"] = tool_calls_with_server_calls
|
||||
# Include model information from the response if available
|
||||
if hasattr(response, "model") and response.model:
|
||||
inputs["model"] = response.model
|
||||
|
|
@ -469,8 +481,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
returned_tool_calls: Final = guardrailed_inputs.get("tool_calls")
|
||||
guardrailed_tool_calls: Final[list[dict[str, object]]] = (
|
||||
cast(list[dict[str, object]], returned_tool_calls)
|
||||
if isinstance(returned_tool_calls, list) and len(returned_tool_calls) == len(tool_calls_to_check)
|
||||
else tool_calls_to_check
|
||||
if isinstance(returned_tool_calls, list)
|
||||
and len(returned_tool_calls) == len(tool_calls_with_server_calls)
|
||||
else tool_calls_with_server_calls
|
||||
)
|
||||
|
||||
# Step 3: Map guardrail responses back to original response structure
|
||||
|
|
@ -488,6 +501,11 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
tool_calls=guardrailed_tool_calls,
|
||||
task_mappings=tool_call_task_mappings,
|
||||
)
|
||||
self._apply_guardrail_responses_to_output_server_tool_calls(
|
||||
response=response,
|
||||
guardrailed_tool_calls=guardrailed_tool_calls[len(tool_calls_to_check) :],
|
||||
task_mappings=tuple(mapping for mapping, _ in server_tool_call_tasks),
|
||||
)
|
||||
|
||||
verbose_proxy_logger.debug("OpenAI Chat Completions: Processed output response: %s", response)
|
||||
|
||||
|
|
@ -681,6 +699,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
pre_guardrail_tool_calls=pre_guardrail_tool_calls,
|
||||
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
|
||||
)
|
||||
self._write_ended_stream_server_tool_call_rewrites(
|
||||
responses_so_far=responses_so_far,
|
||||
guardrailed_response=model_response,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _rebuild_ended_stream_per_choice(
|
||||
|
|
@ -939,6 +961,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
if choice.message.tool_calls and isinstance(choice.message.tool_calls, list):
|
||||
if len(choice.message.tool_calls) > 0:
|
||||
return True
|
||||
if anthropic_server_tool_call_params(choice.message):
|
||||
return True
|
||||
elif isinstance(response, ModelResponseStream):
|
||||
for streaming_choice in response.choices:
|
||||
if isinstance(streaming_choice, litellm.StreamingChoices):
|
||||
|
|
@ -949,6 +973,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
if streaming_choice.delta.tool_calls and isinstance(streaming_choice.delta.tool_calls, list):
|
||||
if len(streaming_choice.delta.tool_calls) > 0:
|
||||
return True
|
||||
if anthropic_server_tool_call_params(streaming_choice.delta):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _extract_output_text_images_and_tool_calls(
|
||||
|
|
@ -1099,6 +1125,25 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
if "name" in func_dict:
|
||||
existing_tool_call.function.name = func_dict["name"]
|
||||
|
||||
@staticmethod
|
||||
def _apply_guardrail_responses_to_output_server_tool_calls(
|
||||
response: "ModelResponse",
|
||||
guardrailed_tool_calls: Sequence[object],
|
||||
task_mappings: Sequence[tuple[int, int]],
|
||||
) -> None:
|
||||
guardrailed_by_task: Final = MappingProxyType(dict(zip(task_mappings, guardrailed_tool_calls)))
|
||||
for choice_idx, choice in enumerate(response.choices):
|
||||
server_tool_calls = anthropic_server_tool_call_params(choice.message)
|
||||
rewritten = tuple(
|
||||
_rewritten_server_tool_call(server_tool_call, guardrailed_by_task.get((choice_idx, server_idx)))
|
||||
for server_idx, server_tool_call in enumerate(server_tool_calls)
|
||||
)
|
||||
if rewritten != server_tool_calls:
|
||||
choice.message.provider_specific_fields = {
|
||||
**(choice.message.provider_specific_fields or {}),
|
||||
"server_tool_calls": list(rewritten),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _string_choice_contents(response: "ModelResponse") -> tuple[str | None, ...]:
|
||||
return tuple(
|
||||
|
|
@ -1211,6 +1256,32 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
for fragment in tail:
|
||||
fragment.function.arguments = ""
|
||||
|
||||
@staticmethod
|
||||
def _write_ended_stream_server_tool_call_rewrites(
|
||||
responses_so_far: Sequence["ModelResponseStream"],
|
||||
guardrailed_response: "ModelResponse",
|
||||
) -> None:
|
||||
guardrailed_by_id: Final = MappingProxyType(
|
||||
{
|
||||
(guardrailed_response.choices[choice_position].index, server_tool_call["id"]): server_tool_call
|
||||
for (choice_position, _), server_tool_call in _server_tool_call_tasks(guardrailed_response.choices)
|
||||
}
|
||||
)
|
||||
if not guardrailed_by_id:
|
||||
return
|
||||
for response in responses_so_far:
|
||||
for choice in response.choices:
|
||||
streamed = anthropic_server_tool_call_params(choice.delta)
|
||||
rewritten = tuple(
|
||||
guardrailed_by_id.get((choice.index, server_tool_call["id"]), server_tool_call)
|
||||
for server_tool_call in streamed
|
||||
)
|
||||
if rewritten != streamed:
|
||||
choice.delta.provider_specific_fields = {
|
||||
**(choice.delta.provider_specific_fields or {}),
|
||||
"server_tool_calls": list(rewritten),
|
||||
}
|
||||
|
||||
async def _apply_guardrail_responses_to_output_streaming(
|
||||
self,
|
||||
responses: list["ModelResponseStream"],
|
||||
|
|
@ -1400,7 +1471,42 @@ def _stream_chunk_choices(item: object) -> Sequence[object]:
|
|||
def _streamed_delta_tool_calls(delta: object) -> tuple[object, ...]:
|
||||
function_call: Final = stream_item_field(delta, "function_call")
|
||||
legacy: Final = () if function_call is None else (function_call,)
|
||||
return stream_item_items(delta, "tool_calls") + legacy
|
||||
return anthropic_server_tool_call_params(delta) + stream_item_items(delta, "tool_calls") + legacy
|
||||
|
||||
|
||||
def _server_tool_call_tasks(
|
||||
choices: Sequence[object],
|
||||
) -> Iterator[tuple[tuple[int, int], ChatCompletionAssistantToolCall]]:
|
||||
for choice_idx, choice in enumerate(choices):
|
||||
if not isinstance(choice, Choices):
|
||||
continue
|
||||
for server_idx, server_tool_call in enumerate(anthropic_server_tool_call_params(choice.message)):
|
||||
yield (choice_idx, server_idx), server_tool_call
|
||||
|
||||
|
||||
def _server_tool_call_input(server_tool_call: ChatCompletionAssistantToolCall) -> dict[str, object]:
|
||||
return {
|
||||
"id": server_tool_call["id"],
|
||||
"type": server_tool_call["type"],
|
||||
"function": dict(server_tool_call["function"]),
|
||||
}
|
||||
|
||||
|
||||
_TOOL_CALL_FUNCTION_ADAPTER: Final = TypeAdapter(ChatCompletionToolCallFunctionChunk)
|
||||
|
||||
|
||||
def _rewritten_server_tool_call(
|
||||
server_tool_call: ChatCompletionAssistantToolCall, guardrailed: object
|
||||
) -> ChatCompletionAssistantToolCall:
|
||||
try:
|
||||
function: Final = _TOOL_CALL_FUNCTION_ADAPTER.validate_python(stream_item_field(guardrailed, "function"))
|
||||
except ValidationError:
|
||||
return server_tool_call
|
||||
return ChatCompletionAssistantToolCall(
|
||||
id=server_tool_call["id"],
|
||||
type=server_tool_call["type"],
|
||||
function={**server_tool_call["function"], **function},
|
||||
)
|
||||
|
||||
|
||||
def _released_choices(responses_so_far: Sequence[object]) -> Iterator[object]:
|
||||
|
|
|
|||
|
|
@ -63,11 +63,26 @@ def response_tool_call_names(completion_response: object) -> tuple[str, ...]:
|
|||
)
|
||||
return tuple(
|
||||
stripped
|
||||
for name in chain(called_names, _auto_executed_mcp_tool_names(completion_response))
|
||||
for name in chain(
|
||||
_server_tool_call_names(completion_response),
|
||||
called_names,
|
||||
_auto_executed_mcp_tool_names(completion_response),
|
||||
)
|
||||
if isinstance(name, str) and (stripped := name.strip())
|
||||
)
|
||||
|
||||
|
||||
def _server_tool_call_names(completion_response: object) -> tuple[str | None, ...]:
|
||||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
|
||||
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
|
||||
|
||||
server_tool_calls: Final = chain.from_iterable(
|
||||
anthropic_server_tool_call_params(message_field(choice, "message"))
|
||||
for choice in parts_of(message_field(completion_response, "choices"))
|
||||
)
|
||||
return tuple(server_tool_call["function"].get("name") for server_tool_call in server_tool_calls)
|
||||
|
||||
|
||||
class _ExecutedTool(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from litellm.integrations.custom_guardrail import (
|
|||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params, anthropic_server_tool_calls
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.callback_utils import (
|
||||
add_guardrail_to_applied_guardrails_header,
|
||||
|
|
@ -515,7 +516,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
|
|||
|
||||
for choice_index, choice in enumerate(response.choices):
|
||||
if isinstance(choice, Choices):
|
||||
for tool in choice.message.tool_calls or []:
|
||||
for tool in (*(choice.message.tool_calls or ()), *anthropic_server_tool_calls(choice.message)):
|
||||
tool_calls.append(tool)
|
||||
legacy_tool_call = self._legacy_function_call_to_tool_call(
|
||||
getattr(choice.message, "function_call", None), choice_index
|
||||
|
|
@ -784,6 +785,23 @@ class ToolPermissionGuardrail(CustomGuardrail):
|
|||
|
||||
choice.message.tool_calls = filtered_tool_calls if filtered_tool_calls else None
|
||||
|
||||
if (server_tool_calls := anthropic_server_tool_call_params(choice.message)) and any(
|
||||
server_tool_call["id"] in error_results for server_tool_call in server_tool_calls
|
||||
):
|
||||
error_messages.extend(
|
||||
error_results[server_tool_call["id"]].content
|
||||
for server_tool_call in server_tool_calls
|
||||
if server_tool_call["id"] in error_results
|
||||
)
|
||||
choice.message.provider_specific_fields = {
|
||||
**(choice.message.provider_specific_fields or {}),
|
||||
"server_tool_calls": [
|
||||
server_tool_call
|
||||
for server_tool_call in server_tool_calls
|
||||
if server_tool_call["id"] not in error_results
|
||||
],
|
||||
}
|
||||
|
||||
legacy_tool_call = self._legacy_function_call_to_tool_call(
|
||||
getattr(choice.message, "function_call", None), choice_index
|
||||
)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import litellm
|
|||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items
|
||||
from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache
|
||||
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend
|
||||
from litellm.llms.anthropic.common_utils import anthropic_server_tool_calls
|
||||
from litellm.main import stream_chunk_builder
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import compaction_output_item
|
||||
from litellm.responses.litellm_completion_transformation.custom_tools import (
|
||||
|
|
@ -486,13 +487,16 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
|
||||
try:
|
||||
message: Final = litellm_complete_object.choices[0].message
|
||||
tool_calls = getattr(message, "tool_calls", None)
|
||||
server_tool_calls: Final = anthropic_server_tool_calls(message)
|
||||
tool_calls: Final = [*server_tool_calls, *(getattr(message, "tool_calls", None) or ())]
|
||||
except Exception:
|
||||
tool_calls = None
|
||||
|
||||
if not tool_calls or not isinstance(tool_calls, list):
|
||||
return
|
||||
|
||||
if not tool_calls:
|
||||
return
|
||||
|
||||
server_tool_call_ids: Final = frozenset(tool_call.id for tool_call in server_tool_calls)
|
||||
|
||||
for tc in tool_calls:
|
||||
call_id_raw = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)
|
||||
if not call_id_raw:
|
||||
|
|
@ -518,6 +522,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
if (search_execution := self._tool_search_execution(call_id, fn_name)) is not None:
|
||||
self._queue_tool_search_call_done(call_id, output_index, fn_args, search_execution)
|
||||
continue
|
||||
if call_id in server_tool_call_ids:
|
||||
continue
|
||||
|
||||
# Track if this is a new tool call that wasn't streamed
|
||||
is_new_tool_call = call_id not in self._tool_item_id_by_call_id
|
||||
|
|
@ -592,10 +598,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
}
|
||||
),
|
||||
)
|
||||
finished_events: Final = (
|
||||
()
|
||||
if item.status == "in_progress"
|
||||
else ((ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),)
|
||||
)
|
||||
for event_type, event_class in (
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, WebSearchCallInProgressEvent),
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, WebSearchCallSearchingEvent),
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),
|
||||
*finished_events,
|
||||
):
|
||||
self._sequence_number += 1
|
||||
event = event_class(type=event_type, output_index=output_index, item_id=item.id)
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ from openai.types.chat.chat_completion_named_tool_choice_param import (
|
|||
from openai.types.responses import ResponseFunctionToolCall, ResponseFunctionWebSearch
|
||||
from openai.types.responses.response_compaction_item import ResponseCompactionItem
|
||||
from openai.types.responses.response_create_params import ResponseInputParam
|
||||
from openai.types.responses.response_function_web_search import ActionOpenPage, ActionSearch
|
||||
from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam
|
||||
from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam
|
||||
from openai.types.responses.tool_param import FunctionToolParam
|
||||
|
|
@ -56,7 +57,12 @@ from litellm.litellm_core_utils.hidden_params import (
|
|||
set_hidden_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
anthropic_server_tool_call_params,
|
||||
anthropic_server_tool_calls,
|
||||
supports_anthropic_cache_control,
|
||||
validated_object_mapping,
|
||||
)
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support
|
||||
from litellm.llms.openrouter.chat.transformation import openrouter_model_supports_cache_control
|
||||
|
|
@ -66,11 +72,12 @@ from litellm.responses.litellm_completion_transformation.session_handler import
|
|||
ResponsesSessionHandler,
|
||||
StoredTurns,
|
||||
)
|
||||
from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, AnthropicToolSearchToolRegex
|
||||
from litellm.types.llms.base import CachedTokensDetails
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionAssistantMessage,
|
||||
ChatCompletionAssistantToolCall,
|
||||
ChatCompletionAudioObject,
|
||||
ChatCompletionCachedContent,
|
||||
ChatCompletionFileObject,
|
||||
|
|
@ -114,6 +121,7 @@ from litellm.types.responses.main import (
|
|||
OutputImageGenerationCall,
|
||||
OutputText,
|
||||
OutputToolSearchCall,
|
||||
web_search_call_tool_id,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionAnnotation,
|
||||
|
|
@ -873,6 +881,29 @@ def _chain_session_metadata(chain_session: str, request_kwargs: Mapping[str, obj
|
|||
return {key: {**_STR_KEY_DICT_ADAPTER.validate_python(metadata), CALLER_SESSION_ID_METADATA_KEY: chain_session}}
|
||||
|
||||
|
||||
def _pending_claude_server_tool_call(search: ResponseFunctionWebSearch) -> ChatCompletionAssistantToolCall | None:
|
||||
tool_id: Final = web_search_call_tool_id(search.id)
|
||||
if search.status != "in_progress" or tool_id is None:
|
||||
return None
|
||||
if not tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX):
|
||||
return None
|
||||
match search.action:
|
||||
case ActionSearch(query=str() as query):
|
||||
return _server_tool_call(tool_id, "web_search", json.dumps({"query": query}))
|
||||
case ActionOpenPage(url=str() as url):
|
||||
return _server_tool_call(tool_id, "web_fetch", json.dumps({"url": url}))
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _server_tool_call(tool_id: str, name: str, arguments: str) -> ChatCompletionAssistantToolCall:
|
||||
return ChatCompletionAssistantToolCall(
|
||||
id=tool_id,
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(name=name, arguments=arguments),
|
||||
)
|
||||
|
||||
|
||||
class LiteLLMCompletionResponsesConfig:
|
||||
@staticmethod
|
||||
def get_supported_openai_params(model: str) -> list:
|
||||
|
|
@ -1778,10 +1809,22 @@ class LiteLLMCompletionResponsesConfig:
|
|||
message_cache_control: Final = (
|
||||
None if isinstance(content, list) else new_cache_control or previous_cache_control
|
||||
)
|
||||
new_server_tool_calls: Final = anthropic_server_tool_call_params(new_message)
|
||||
server_tool_call_fields: Final = (
|
||||
{
|
||||
"provider_specific_fields": {
|
||||
**(validated_object_mapping(last_message.get("provider_specific_fields")) or {}),
|
||||
"server_tool_calls": [*anthropic_server_tool_call_params(last_message), *new_server_tool_calls],
|
||||
}
|
||||
}
|
||||
if new_server_tool_calls
|
||||
else {}
|
||||
)
|
||||
merged: Final = {
|
||||
**{key: value for key, value in last_message.items() if key != "cache_control"},
|
||||
"content": content,
|
||||
**({} if message_cache_control is None else {"cache_control": message_cache_control}),
|
||||
**server_tool_call_fields,
|
||||
}
|
||||
return cast( # cast-ok: preserves the assistant fields and content blocks
|
||||
ChatCompletionAssistantMessage, merged
|
||||
|
|
@ -2223,10 +2266,15 @@ class LiteLLMCompletionResponsesConfig:
|
|||
content="Hosted web search: " + replay.model_dump_json(exclude_none=True),
|
||||
)
|
||||
]
|
||||
hosted_search_text: Final = "Hosted web search: " + search.model_dump_json(exclude_none=True)
|
||||
pending_server_tool_call: Final = _pending_claude_server_tool_call(search)
|
||||
if pending_server_tool_call is None:
|
||||
return [GenericChatCompletionMessage(role="assistant", content=hosted_search_text)]
|
||||
return [
|
||||
GenericChatCompletionMessage(
|
||||
ChatCompletionResponseMessage(
|
||||
role="assistant",
|
||||
content="Hosted web search: " + search.model_dump_json(exclude_none=True),
|
||||
content=hosted_search_text,
|
||||
provider_specific_fields={"server_tool_calls": [pending_server_tool_call]},
|
||||
)
|
||||
]
|
||||
if LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(input_item):
|
||||
|
|
@ -2956,13 +3004,12 @@ class LiteLLMCompletionResponsesConfig:
|
|||
all_chat_completion_tools: Final[list[ChatCompletionMessageToolCall]] = []
|
||||
for choice in chat_completion_response.choices:
|
||||
if isinstance(choice, Choices):
|
||||
if choice.message.tool_calls:
|
||||
all_chat_completion_tools.extend(choice.message.tool_calls)
|
||||
for tool_call in choice.message.tool_calls:
|
||||
TOOL_CALLS_CACHE.set_cache(
|
||||
key=tool_call.id,
|
||||
value=tool_call,
|
||||
)
|
||||
for tool_call in (*anthropic_server_tool_calls(choice.message), *(choice.message.tool_calls or ())):
|
||||
all_chat_completion_tools.append(tool_call)
|
||||
TOOL_CALLS_CACHE.set_cache(
|
||||
key=tool_call.id,
|
||||
value=tool_call,
|
||||
)
|
||||
|
||||
request_tools: Final = responses_api_request.get("tools") if responses_api_request is not None else None
|
||||
custom_tool_names: Final = extract_custom_tool_names(request_tools)
|
||||
|
|
@ -3379,7 +3426,22 @@ class LiteLLMCompletionResponsesConfig:
|
|||
for item in responses_output
|
||||
]
|
||||
|
||||
return responses_output
|
||||
server_tool_call_ids: Final = LiteLLMCompletionResponsesConfig._server_tool_call_ids(choices)
|
||||
return [
|
||||
item
|
||||
for item in responses_output
|
||||
if not (
|
||||
getattr(item, "type", None) == "function_call"
|
||||
and getattr(item, "call_id", None) in server_tool_call_ids
|
||||
)
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _server_tool_call_ids(choices: Sequence[Choices]) -> frozenset[str]:
|
||||
server_tool_calls: Final = chain.from_iterable(
|
||||
anthropic_server_tool_calls(choice.message) for choice in choices
|
||||
)
|
||||
return frozenset(tool_call.id for tool_call in server_tool_calls)
|
||||
|
||||
@staticmethod
|
||||
def _extract_tool_result_output_items(
|
||||
|
|
|
|||
|
|
@ -120,9 +120,12 @@ def _replays_provider_specific_fields(
|
|||
client_tool_calls: tuple[ChatCompletionMessageToolCall, ...],
|
||||
) -> bool:
|
||||
from litellm.completion_extras.litellm_responses_transformation.transformation import carries_tool_search_items
|
||||
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
|
||||
|
||||
return client_tool_calls != tool_calls or carries_tool_search_items(
|
||||
getattr(message, "provider_specific_fields", None)
|
||||
return (
|
||||
client_tool_calls != tool_calls
|
||||
or bool(anthropic_server_tool_call_params(message))
|
||||
or carries_tool_search_items(getattr(message, "provider_specific_fields", None))
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,12 @@ from collections.abc import Mapping, Sequence
|
|||
from typing import Final, Literal, Optional, Union
|
||||
|
||||
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
|
||||
from openai.types.responses.response_function_web_search import ActionSearchSource, ResponseFunctionWebSearch
|
||||
from openai.types.responses.response_function_web_search import (
|
||||
ActionOpenPage,
|
||||
ActionSearch,
|
||||
ActionSearchSource,
|
||||
ResponseFunctionWebSearch,
|
||||
)
|
||||
from pydantic import ConfigDict, PrivateAttr
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
|
|
@ -42,14 +47,11 @@ class OutputFunctionToolCall(BaseLiteLLMOpenAIResponseObject):
|
|||
phase: Phase = None
|
||||
|
||||
|
||||
def build_web_search_call(
|
||||
tool_id: str,
|
||||
tool_input: object,
|
||||
result: object,
|
||||
status: Literal["in_progress", "searching", "completed", "failed"] | None = None,
|
||||
) -> ResponseFunctionWebSearch:
|
||||
HOSTED_WEB_TOOL_NAMES: Final = frozenset({"web_search", "web_fetch"})
|
||||
|
||||
|
||||
def _web_search_action(tool_input: object, content: object) -> ActionSearch:
|
||||
query: Final = tool_input.get("query", "") if isinstance(tool_input, Mapping) else ""
|
||||
content: Final = result.get("content") if isinstance(result, Mapping) else None
|
||||
result_items: Final = content if isinstance(content, Sequence) and not isinstance(content, (str, bytes)) else ()
|
||||
sources: Final = [
|
||||
ActionSearchSource(type="url", url=url)
|
||||
|
|
@ -58,20 +60,62 @@ def build_web_search_call(
|
|||
and item.get("type") == "web_search_result"
|
||||
and isinstance((url := item.get("url")), str)
|
||||
]
|
||||
failed: Final = isinstance(content, Mapping) and content.get("type") == "web_search_tool_result_error"
|
||||
return ResponseFunctionWebSearch(
|
||||
id=f"ws_{tool_id}",
|
||||
type="web_search_call",
|
||||
status=status or ("failed" if failed else "completed"),
|
||||
action={
|
||||
"type": "search",
|
||||
"query": query if isinstance(query, str) else "",
|
||||
"queries": [query] if isinstance(query, str) and query else [],
|
||||
"sources": sources,
|
||||
},
|
||||
return ActionSearch(
|
||||
type="search",
|
||||
query=query if isinstance(query, str) else "",
|
||||
queries=[query] if isinstance(query, str) and query else [],
|
||||
sources=sources,
|
||||
)
|
||||
|
||||
|
||||
def _web_fetch_action(tool_input: object) -> ActionOpenPage:
|
||||
match tool_input:
|
||||
case {"url": str() as url}:
|
||||
return ActionOpenPage(type="open_page", url=url)
|
||||
case _:
|
||||
return ActionOpenPage(type="open_page", url=None)
|
||||
|
||||
|
||||
def _result_content(result: object) -> object:
|
||||
match result:
|
||||
case {"content": object() as content}:
|
||||
return content
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _is_tool_result_error(content: object, tool_name: str) -> bool:
|
||||
match content:
|
||||
case {"type": str() as content_type}:
|
||||
return content_type == f"{tool_name}_tool_result_error"
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
WEB_SEARCH_CALL_ID_PREFIX: Final = "ws_"
|
||||
|
||||
|
||||
def build_web_search_call(
|
||||
tool_id: str,
|
||||
tool_input: object,
|
||||
result: object,
|
||||
status: Literal["in_progress", "searching", "completed", "failed"] | None = None,
|
||||
tool_name: str = "web_search",
|
||||
) -> ResponseFunctionWebSearch:
|
||||
content: Final = _result_content(result)
|
||||
return ResponseFunctionWebSearch(
|
||||
id=f"{WEB_SEARCH_CALL_ID_PREFIX}{tool_id}",
|
||||
type="web_search_call",
|
||||
status=status or ("failed" if _is_tool_result_error(content, tool_name) else "completed"),
|
||||
action=_web_fetch_action(tool_input) if tool_name == "web_fetch" else _web_search_action(tool_input, content),
|
||||
)
|
||||
|
||||
|
||||
def web_search_call_tool_id(call_id: str) -> str | None:
|
||||
tool_id: Final = call_id.removeprefix(WEB_SEARCH_CALL_ID_PREFIX)
|
||||
return None if tool_id == call_id else tool_id
|
||||
|
||||
|
||||
class OutputImageGenerationCall(BaseLiteLLMOpenAIResponseObject):
|
||||
"""An image generation call output"""
|
||||
|
||||
|
|
|
|||
|
|
@ -110,6 +110,41 @@ def test_output_tool_calls_use_the_datadog_tool_call_schema(logger: DataDogLLMOb
|
|||
assert "function" not in message["tool_calls"][0]
|
||||
|
||||
|
||||
|
||||
def test_claude_server_tool_calls_are_reported_before_client_tool_calls(logger: DataDogLLMObsLogger) -> None:
|
||||
payload = build(
|
||||
logger,
|
||||
response_message={
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [ASSISTANT_TOOL_CALL],
|
||||
"provider_specific_fields": {
|
||||
"server_tool_calls": [
|
||||
{
|
||||
"id": "srvtoolu_01Search",
|
||||
"type": "function",
|
||||
"function": {"name": "web_search", "arguments": '{"query": "paris weather"}'},
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
assert payload["meta"]["output"]["messages"][0]["tool_calls"] == [
|
||||
{
|
||||
"name": "web_search",
|
||||
"arguments": {"query": "paris weather"},
|
||||
"tool_id": "srvtoolu_01Search",
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"name": "get_weather",
|
||||
"arguments": {"city": "Paris", "unit": "c"},
|
||||
"tool_id": "call_abc123",
|
||||
"type": "function",
|
||||
},
|
||||
]
|
||||
|
||||
def test_tool_call_identifiers_that_are_not_strings_are_blanked_not_stringified(logger: DataDogLLMObsLogger) -> None:
|
||||
payload = build(
|
||||
logger,
|
||||
|
|
|
|||
|
|
@ -174,6 +174,45 @@ def test_openinference_output_tool_calls_preserve_calls_in_attributes_and_value(
|
|||
]
|
||||
|
||||
|
||||
def test_openinference_output_lists_claude_server_tool_calls_before_client_tool_calls():
|
||||
data: Final = _llm_call(
|
||||
choices_out=(
|
||||
{
|
||||
"finish_reason": "tool_calls",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_paris",
|
||||
"type": "function",
|
||||
"function": {"name": "lookup_weather", "arguments": '{"city": "Paris"}'},
|
||||
}
|
||||
],
|
||||
"provider_specific_fields": {
|
||||
"server_tool_calls": [
|
||||
{
|
||||
"id": "srvtoolu_01Search",
|
||||
"type": "function",
|
||||
"function": {"name": "web_search", "arguments": '{"query": "paris weather"}'},
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
)
|
||||
attrs: Final = OpenInferenceMapper().map(data)
|
||||
assert {key: value for key, value in attrs.items() if ".tool_calls." in key} == {
|
||||
"llm.output_messages.0.message.tool_calls.0.tool_call.id": "srvtoolu_01Search",
|
||||
"llm.output_messages.0.message.tool_calls.0.tool_call.function.name": "web_search",
|
||||
"llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": '{"query": "paris weather"}',
|
||||
"llm.output_messages.0.message.tool_calls.1.tool_call.id": "call_paris",
|
||||
"llm.output_messages.0.message.tool_calls.1.tool_call.function.name": "lookup_weather",
|
||||
"llm.output_messages.0.message.tool_calls.1.tool_call.function.arguments": '{"city": "Paris"}',
|
||||
}
|
||||
|
||||
|
||||
def test_openinference_responses_tool_calls_are_emitted_as_output_attributes():
|
||||
data: Final = LLMCallSpanData.from_standard_logging_payload(
|
||||
_responses_payload(
|
||||
|
|
|
|||
|
|
@ -15,7 +15,9 @@ import litellm
|
|||
from litellm._uuid import uuid
|
||||
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
|
||||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
|
||||
from litellm.llms.anthropic.chat.handler import ModelResponseIterator, _envelope_verdict, make_call, make_sync_call
|
||||
from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params
|
||||
from litellm.llms.base_llm.base_utils import response_format_schema, unwrap_response_format_envelope
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -23,7 +25,7 @@ from litellm.types.llms.openai import (
|
|||
ChatCompletionToolCallFunctionChunk,
|
||||
)
|
||||
from litellm.types.responses.main import OutputCodeInterpreterCall
|
||||
from litellm.types.utils import ChatCompletionDeltaToolCall, ModelResponseStream
|
||||
from litellm.types.utils import ChatCompletionDeltaToolCall, ChatCompletionMessageToolCall, Choices, ModelResponseStream
|
||||
from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE
|
||||
|
||||
|
||||
|
|
@ -1780,19 +1782,113 @@ def test_web_search_tool_result_no_extra_tool_calls():
|
|||
for tc in parsed.choices[0].delta.tool_calls:
|
||||
tool_calls_emitted.append(tc)
|
||||
|
||||
# Should have exactly 2 tool calls:
|
||||
# 1. From content_block_start (server_tool_use) with id and name
|
||||
# 2. From content_block_delta with the actual query
|
||||
assert len(tool_calls_emitted) == 2, f"Expected 2 tool calls, got {len(tool_calls_emitted)}"
|
||||
assert tool_calls_emitted == []
|
||||
|
||||
# First tool call should have the id and name
|
||||
assert tool_calls_emitted[0]["id"] == "srvtoolu_01ABC123"
|
||||
assert tool_calls_emitted[0]["function"]["name"] == "web_search"
|
||||
|
||||
# Second tool call should have the query arguments
|
||||
assert tool_calls_emitted[1]["function"]["arguments"] == '{"query": "test"}'
|
||||
def _server_tool_input_delta(fragment: str) -> dict[str, object]:
|
||||
return {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": fragment}}
|
||||
|
||||
# The {} chunks from web_search_tool_result should NOT have been emitted as tool calls
|
||||
|
||||
def _server_search_then_answer(*partial_json: str) -> tuple[dict[str, object], ...]:
|
||||
return (
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {"id": "msg_01", "model": "claude-opus-5-5", "usage": {"input_tokens": 9}},
|
||||
},
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 0,
|
||||
"content_block": {"type": "server_tool_use", "id": "srvtoolu_01Web", "name": "web_search", "input": {}},
|
||||
},
|
||||
*(_server_tool_input_delta(fragment) for fragment in partial_json),
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 1,
|
||||
"content_block": {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_01Web", "content": []},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 1},
|
||||
{"type": "content_block_start", "index": 2, "content_block": {"type": "text", "text": ""}},
|
||||
{"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": "Paris is in France."}},
|
||||
{"type": "content_block_stop", "index": 2},
|
||||
)
|
||||
|
||||
|
||||
_SERVER_SEARCH_THEN_ANSWER: Final = _server_search_then_answer('{"query": "Paris"}')
|
||||
_CLIENT_TOOL_CALL: Final[tuple[dict[str, object], ...]] = (
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 3,
|
||||
"content_block": {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {}},
|
||||
},
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": 3,
|
||||
"delta": {"type": "input_json_delta", "partial_json": '{"city": "Paris"}'},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 3},
|
||||
)
|
||||
|
||||
|
||||
def _message_delta(stop_reason: str) -> dict[str, object]:
|
||||
return {"type": "message_delta", "delta": {"stop_reason": stop_reason}, "usage": {"output_tokens": 20}}
|
||||
|
||||
|
||||
_PARIS_SEARCH: Final = ("srvtoolu_01Web", "web_search", '{"query": "Paris"}')
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("events", "expected_tool_calls", "expected_finish_reason", "expected_server_tool_call"),
|
||||
[
|
||||
pytest.param(
|
||||
(*_SERVER_SEARCH_THEN_ANSWER, _message_delta("end_turn")), [], "stop", _PARIS_SEARCH, id="server_tool_only"
|
||||
),
|
||||
pytest.param(
|
||||
(*_SERVER_SEARCH_THEN_ANSWER, *_CLIENT_TOOL_CALL, _message_delta("tool_use")),
|
||||
[("toolu_01Weather", "get_weather", '{"city": "Paris"}')],
|
||||
"tool_calls",
|
||||
_PARIS_SEARCH,
|
||||
id="server_tool_then_client_tool",
|
||||
),
|
||||
pytest.param(
|
||||
(*_server_search_then_answer('{"que', 'ry": "Pa', 'ris"}'), _message_delta("end_turn")),
|
||||
[],
|
||||
"stop",
|
||||
_PARIS_SEARCH,
|
||||
id="server_tool_input_split_across_deltas",
|
||||
),
|
||||
pytest.param(
|
||||
(*_server_search_then_answer(), _message_delta("end_turn")),
|
||||
[],
|
||||
"stop",
|
||||
("srvtoolu_01Web", "web_search", "{}"),
|
||||
id="server_tool_with_empty_input",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_streamed_server_tool_use_is_not_returned_as_a_client_tool_call(
|
||||
events: tuple[dict[str, object], ...],
|
||||
expected_tool_calls: list[tuple[str, str, str]],
|
||||
expected_finish_reason: str,
|
||||
expected_server_tool_call: tuple[str, str, str],
|
||||
):
|
||||
choice: Final = _assemble_stream(events).choices[0]
|
||||
assert isinstance(choice, Choices)
|
||||
|
||||
assert (choice.message.tool_calls is None) == (not expected_tool_calls)
|
||||
assert [
|
||||
(call.id, call.function.name, call.function.arguments)
|
||||
for call in choice.message.tool_calls or ()
|
||||
if isinstance(call, ChatCompletionMessageToolCall)
|
||||
] == expected_tool_calls
|
||||
assert choice.finish_reason == expected_finish_reason
|
||||
assert choice.message.content == "Paris is in France."
|
||||
assert [
|
||||
(call["id"], call["function"].get("name"), call["function"].get("arguments"))
|
||||
for call in anthropic_server_tool_call_params(choice.message)
|
||||
] == [expected_server_tool_call]
|
||||
web_search_results: Final = message_field(choice.message.provider_specific_fields, "web_search_results")
|
||||
assert [message_field(result, "tool_use_id") for result in parts_of(web_search_results)] == ["srvtoolu_01Web"]
|
||||
|
||||
|
||||
def test_current_content_block_type_tracking():
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import copy
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
from typing import Final, cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -22,8 +23,13 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
|
||||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
|
||||
from litellm.llms.anthropic.chat import ModelResponseIterator
|
||||
from litellm.llms.anthropic.common_utils import process_anthropic_headers
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
anthropic_server_tool_call_params,
|
||||
process_anthropic_headers,
|
||||
validated_object_mapping,
|
||||
)
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
|
|
@ -38,8 +44,8 @@ from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation im
|
|||
VertexAIAnthropicConfig,
|
||||
)
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES
|
||||
from litellm.types.llms.openai import ChatCompletionToolCallFunctionChunk
|
||||
from litellm.types.utils import ChatCompletionToolCallChunk, ModelResponse, ServerToolUse, Usage
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallFunctionChunk
|
||||
from litellm.types.utils import ChatCompletionToolCallChunk, Choices, Message, ModelResponse, ServerToolUse, Usage
|
||||
from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE
|
||||
|
||||
|
||||
|
|
@ -3509,21 +3515,17 @@ def test_code_execution_tool_results_extraction():
|
|||
prefix_prompt=None,
|
||||
)
|
||||
|
||||
# Verify tool calls are present
|
||||
assert transformed_response.choices[0].message.tool_calls is not None
|
||||
assert len(transformed_response.choices[0].message.tool_calls) == 2
|
||||
assert transformed_response.choices[0].message.tool_calls is None
|
||||
|
||||
# Verify first tool call
|
||||
assert transformed_response.choices[0].message.tool_calls[0].id == "srvtoolu_01ABC"
|
||||
assert transformed_response.choices[0].message.tool_calls[0].function.name == "bash_code_execution"
|
||||
|
||||
# Verify second tool call
|
||||
assert transformed_response.choices[0].message.tool_calls[1].id == "srvtoolu_01DEF"
|
||||
assert transformed_response.choices[0].message.tool_calls[1].function.name == "text_editor_code_execution"
|
||||
|
||||
# Verify tool results are in provider_specific_fields
|
||||
provider_fields = transformed_response.choices[0].message.provider_specific_fields
|
||||
assert provider_fields is not None
|
||||
server_tool_calls: Final = anthropic_server_tool_call_params(transformed_response.choices[0].message)
|
||||
assert [(call["id"], call["function"].get("name")) for call in server_tool_calls] == [
|
||||
("srvtoolu_01ABC", "bash_code_execution"),
|
||||
("srvtoolu_01DEF", "text_editor_code_execution"),
|
||||
]
|
||||
|
||||
# Verify tool results are in provider_specific_fields
|
||||
assert "tool_results" in provider_fields
|
||||
assert provider_fields["tool_results"] is not None
|
||||
assert len(provider_fields["tool_results"]) == 2
|
||||
|
|
@ -3633,7 +3635,7 @@ def _tool_search_turn(tool_references: list[str], last_block: dict[str, object])
|
|||
]
|
||||
|
||||
|
||||
def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str):
|
||||
def _parsed_choice(content: list[dict[str, object]], stop_reason: str) -> Choices:
|
||||
anthropic_response: Final = {
|
||||
"id": "msg_01XYZ",
|
||||
"type": "message",
|
||||
|
|
@ -3648,7 +3650,7 @@ def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str
|
|||
raw_response.json.return_value = anthropic_response
|
||||
raw_response.status_code = 200
|
||||
raw_response.headers = {}
|
||||
return (
|
||||
choice: Final = (
|
||||
AnthropicConfig()
|
||||
.transform_parsed_response(
|
||||
completion_response=anthropic_response,
|
||||
|
|
@ -3658,14 +3660,19 @@ def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str
|
|||
prefix_prompt=None,
|
||||
)
|
||||
.choices[0]
|
||||
.message
|
||||
)
|
||||
assert isinstance(choice, Choices)
|
||||
return choice
|
||||
|
||||
|
||||
def _parsed_assistant_message(content: list[dict[str, object]], stop_reason: str) -> Message:
|
||||
return _parsed_choice(content, stop_reason).message
|
||||
|
||||
|
||||
_WEATHER_CALL: Final = {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}}
|
||||
|
||||
|
||||
def _web_search_turn(last_block: dict[str, object]) -> list[dict[str, object]]:
|
||||
def _web_search_turn(last_block: Mapping[str, object]) -> list[dict[str, object]]:
|
||||
return [
|
||||
{"type": "server_tool_use", "id": "srvtoolu_01Web", "name": "web_search", "input": {"query": "Paris facts"}},
|
||||
{
|
||||
|
|
@ -3681,7 +3688,7 @@ def _web_search_turn(last_block: dict[str, object]) -> list[dict[str, object]]:
|
|||
}
|
||||
],
|
||||
},
|
||||
last_block,
|
||||
dict(last_block),
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -3728,8 +3735,114 @@ def test_server_tool_result_round_trips_right_after_its_call(
|
|||
assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"]
|
||||
|
||||
|
||||
def _block_ids(message: dict[str, object], block_type: str, id_key: str) -> list[object]:
|
||||
return sorted(block[id_key] for block in message["content"] if block["type"] == block_type)
|
||||
_FETCH_STILL_WAITING_ON_ITS_RESULT: Final = {
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_01Fetch",
|
||||
"name": "web_fetch",
|
||||
"input": {"url": "https://example.com/"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("modify_params", [False, True])
|
||||
def test_server_call_still_waiting_on_its_result_is_replayed_as_a_server_call(
|
||||
modify_params: bool, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
content: Final = [_FETCH_STILL_WAITING_ON_ITS_RESULT, _WEATHER_CALL]
|
||||
message: Final = _parsed_assistant_message(content, "tool_use")
|
||||
monkeypatch.setattr(litellm, "modify_params", modify_params)
|
||||
|
||||
replayed: Final = _replayed(
|
||||
{"role": "user", "content": "Fetch example.com, then tell me the weather in Paris"},
|
||||
message.model_dump(),
|
||||
{"role": "tool", "tool_call_id": "toolu_01Weather", "content": "18C"},
|
||||
)
|
||||
|
||||
assert replayed[1]["content"] == content
|
||||
assert [turn["role"] for turn in replayed] == ["user", "assistant", "user"]
|
||||
assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"]
|
||||
|
||||
|
||||
_PARIS_TEXT: Final = "Paris is the capital of France."
|
||||
_PARIS_ANSWER: Final[Mapping[str, str]] = {"type": "text", "text": _PARIS_TEXT}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("content", "stop_reason", "expected_tool_call_ids", "expected_finish_reason"),
|
||||
[
|
||||
pytest.param(_web_search_turn(_PARIS_ANSWER), "end_turn", [], "stop", id="web_search_then_answer"),
|
||||
pytest.param(
|
||||
_web_search_turn(_WEATHER_CALL), "tool_use", ["toolu_01Weather"], "tool_calls", id="web_search_then_tool"
|
||||
),
|
||||
pytest.param([_WEATHER_CALL], "tool_use", ["toolu_01Weather"], "tool_calls", id="client_tool_only"),
|
||||
],
|
||||
)
|
||||
def test_server_tool_use_is_not_returned_as_a_client_tool_call(
|
||||
content: list[dict[str, object]],
|
||||
stop_reason: str,
|
||||
expected_tool_call_ids: list[str],
|
||||
expected_finish_reason: str,
|
||||
):
|
||||
choice: Final = _parsed_choice(content, stop_reason)
|
||||
|
||||
assert (choice.message.tool_calls is None) == (not expected_tool_call_ids)
|
||||
assert [call.id for call in choice.message.tool_calls or ()] == expected_tool_call_ids
|
||||
assert choice.finish_reason == expected_finish_reason
|
||||
assert [
|
||||
(call["id"], call["function"].get("name"), call["function"].get("arguments"))
|
||||
for call in anthropic_server_tool_call_params(choice.message)
|
||||
] == [
|
||||
(block["id"], block["name"], json.dumps(block["input"]))
|
||||
for block in content
|
||||
if block["type"] == "server_tool_use"
|
||||
]
|
||||
if not expected_tool_call_ids:
|
||||
assert choice.message.content == _PARIS_TEXT
|
||||
|
||||
|
||||
def _replayed(*messages: object):
|
||||
return anthropic_messages_pt(
|
||||
messages=cast(list[AllMessageValues], list(messages)), # cast-ok: replays history the way clients send it back
|
||||
model="claude-opus-5-5",
|
||||
llm_provider="anthropic",
|
||||
)
|
||||
|
||||
|
||||
def test_web_search_answer_round_trips_without_any_client_tool_call():
|
||||
message: Final = _parsed_assistant_message(_web_search_turn(_PARIS_ANSWER), "end_turn")
|
||||
|
||||
replayed: Final = _replayed(
|
||||
{"role": "user", "content": "Tell me about Paris"},
|
||||
message.model_dump(),
|
||||
{"role": "user", "content": "And Lyon?"},
|
||||
)
|
||||
|
||||
assert _block_ids(replayed[1], "server_tool_use", "id") == ["srvtoolu_01Web"]
|
||||
assert _block_ids(replayed[1], "web_search_tool_result", "tool_use_id") == ["srvtoolu_01Web"]
|
||||
assert [
|
||||
message_field(block, "text")
|
||||
for block in parts_of(replayed[1]["content"])
|
||||
if message_field(block, "type") == "text"
|
||||
] == [_PARIS_TEXT]
|
||||
|
||||
|
||||
def _block_ids(message: Mapping[str, object], block_type: str, id_key: str) -> list[str]:
|
||||
return sorted(
|
||||
block_id
|
||||
for block in parts_of(message["content"])
|
||||
if message_field(block, "type") == block_type and isinstance(block_id := message_field(block, id_key), str)
|
||||
)
|
||||
|
||||
|
||||
def _as_returned_before_server_tool_calls_moved(message: Message) -> Mapping[str, object]:
|
||||
dumped: Final[Mapping[str, object]] = message.model_dump()
|
||||
provider_fields: Final = validated_object_mapping(dumped.get("provider_specific_fields")) or {}
|
||||
return {
|
||||
**dumped,
|
||||
"tool_calls": [*anthropic_server_tool_call_params(message), *parts_of(dumped.get("tool_calls"))],
|
||||
"provider_specific_fields": {
|
||||
key: value for key, value in provider_fields.items() if key != "server_tool_calls"
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
_SEARCH_THAT_FOUND_NOTHING: Final = _tool_search_turn(
|
||||
|
|
@ -3764,21 +3877,33 @@ def test_replayed_search_that_found_nothing_answers_every_tool_use_and_ends_on_a
|
|||
server_tool_use_ids: list[str],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
|
||||
|
||||
message: Final = _parsed_assistant_message(_SEARCH_THAT_FOUND_NOTHING, "end_turn")
|
||||
monkeypatch.setattr(litellm, "modify_params", modify_params)
|
||||
|
||||
replayed: Final = anthropic_messages_pt(
|
||||
messages=[{"role": "user", "content": "Find a tool and use it"}, message.model_dump(), *follow_ups],
|
||||
model="claude-opus-5-5",
|
||||
llm_provider="anthropic",
|
||||
replayed: Final = _replayed(
|
||||
{"role": "user", "content": "Find a tool and use it"},
|
||||
_as_returned_before_server_tool_calls_moved(message),
|
||||
*follow_ups,
|
||||
)
|
||||
assert [turn["role"] for turn in replayed] == ["user", "assistant", "user"]
|
||||
assert _block_ids(replayed[2], "tool_result", "tool_use_id") == _block_ids(replayed[1], "tool_use", "id")
|
||||
assert _block_ids(replayed[1], "server_tool_use", "id") == server_tool_use_ids
|
||||
|
||||
|
||||
def test_replayed_search_that_found_nothing_keeps_its_result_when_the_call_never_reached_the_client(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
message: Final = _parsed_assistant_message(_SEARCH_THAT_FOUND_NOTHING, "end_turn")
|
||||
monkeypatch.setattr(litellm, "modify_params", True)
|
||||
|
||||
replayed: Final = _replayed({"role": "user", "content": "Find a tool and use it"}, message.model_dump())
|
||||
|
||||
assert [turn["role"] for turn in replayed] == ["user", "assistant"]
|
||||
assert _block_ids(replayed[1], "server_tool_use", "id") == ["srvtoolu_01Search"]
|
||||
assert _block_ids(replayed[1], "tool_search_tool_result", "tool_use_id") == ["srvtoolu_01Search"]
|
||||
assert _block_ids(replayed[1], "tool_use", "id") == []
|
||||
|
||||
|
||||
_CODE_EXECUTION_BLOCKS: Final = [
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
|
|
|
|||
|
|
@ -80,6 +80,59 @@ def test_native_compaction_response_roundtrip(text: str | None, count: int, expe
|
|||
assert result["stop_reason"] == expected_stop
|
||||
|
||||
|
||||
_SERVER_AND_CLIENT_TOOL_TURN: Final[tuple[dict[str, object], ...]] = (
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_01Fetch",
|
||||
"name": "web_fetch",
|
||||
"input": {"url": "https://example.com/"},
|
||||
},
|
||||
{
|
||||
"type": "web_fetch_tool_result",
|
||||
"tool_use_id": "srvtoolu_01Fetch",
|
||||
"content": {"type": "web_fetch_result", "url": "https://example.com/"},
|
||||
},
|
||||
{"type": "server_tool_use", "id": "srvtoolu_02Search", "name": "web_search", "input": {"query": "Paris"}},
|
||||
{"type": "text", "text": "The page is titled Example."},
|
||||
{"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}},
|
||||
)
|
||||
|
||||
|
||||
def test_claude_server_tool_blocks_survive_the_round_trip_through_chat_completions() -> None:
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
|
||||
anthropic_response: Final = {
|
||||
"id": "msg_01Fetch",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5-5",
|
||||
"content": list(_SERVER_AND_CLIENT_TOOL_TURN),
|
||||
"stop_reason": "tool_use",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 100, "output_tokens": 50},
|
||||
}
|
||||
raw_response: Final = MagicMock(spec=httpx.Response)
|
||||
raw_response.json.return_value = anthropic_response
|
||||
raw_response.status_code = 200
|
||||
raw_response.headers = {}
|
||||
chat_response: Final = AnthropicConfig().transform_parsed_response(
|
||||
completion_response=anthropic_response,
|
||||
raw_response=raw_response,
|
||||
model_response=ModelResponse(),
|
||||
json_mode=False,
|
||||
prefix_prompt=None,
|
||||
)
|
||||
|
||||
result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(chat_response)
|
||||
|
||||
assert [(block["type"], block.get("id") or block.get("tool_use_id")) for block in result["content"]] == [
|
||||
(block["type"], block.get("id") or block.get("tool_use_id")) for block in _SERVER_AND_CLIENT_TOOL_TURN
|
||||
]
|
||||
assert result["stop_reason"] == "tool_use"
|
||||
|
||||
|
||||
def test_translate_chat_refusal_to_anthropic_response():
|
||||
response = ModelResponse(
|
||||
id="chatcmpl-refusal",
|
||||
|
|
|
|||
|
|
@ -144,6 +144,42 @@ class MockMisalignedToolCallGuardrail(CustomGuardrail):
|
|||
)
|
||||
|
||||
|
||||
def _server_tool_call(call_id: str, name: str, arguments: Mapping[str, object]) -> dict[str, object]:
|
||||
return {"id": call_id, "type": "function", "function": {"name": name, "arguments": json.dumps(arguments)}}
|
||||
|
||||
|
||||
def _tool_call_ids_names_and_arguments(tool_calls) -> list[tuple[str, str, str]]:
|
||||
return [
|
||||
(tool_call["id"], tool_call["function"]["name"], tool_call["function"]["arguments"])
|
||||
for tool_call in tool_calls
|
||||
]
|
||||
|
||||
|
||||
class QueryRedactingGuardrail(CustomGuardrail):
|
||||
"""Returns copied tool calls with any ``query`` argument redacted, leaving other calls as they came."""
|
||||
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
request_data: dict,
|
||||
input_type: Literal["request", "response"],
|
||||
logging_obj: Optional[Any] = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
def redacted(tool_call):
|
||||
arguments = json.loads(tool_call["function"]["arguments"])
|
||||
if "query" not in arguments:
|
||||
return tool_call
|
||||
return {
|
||||
**tool_call,
|
||||
"function": {**tool_call["function"], "arguments": json.dumps({**arguments, "query": "[REDACTED]"})},
|
||||
}
|
||||
|
||||
return GenericGuardrailAPIInputs(
|
||||
texts=inputs.get("texts", []),
|
||||
tool_calls=[redacted(tool_call) for tool_call in inputs.get("tool_calls", [])],
|
||||
)
|
||||
|
||||
|
||||
class TestOpenAIChatCompletionsHandlerToolsInput:
|
||||
"""Test input processing with tools (function definitions)"""
|
||||
|
||||
|
|
@ -906,6 +942,92 @@ class TestOpenAIChatCompletionsHandlerToolCallsOutput:
|
|||
assert tool_calls[0].function.arguments == first_args
|
||||
assert tool_calls[1].function.arguments == second_args
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_output_guardrail_sees_a_response_that_only_carries_server_tool_calls(self):
|
||||
handler = OpenAIChatCompletionsHandler()
|
||||
guardrail = TestOpenAIChatCompletionsHandlerStreamingOutput._recording_guardrail()
|
||||
response = ModelResponse(
|
||||
model="claude-sonnet-4-5",
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="stop",
|
||||
index=0,
|
||||
message=Message(
|
||||
content=None,
|
||||
role="assistant",
|
||||
provider_specific_fields={
|
||||
"server_tool_calls": [_server_tool_call("srvtoolu_01Search", "web_search", {"query": "x"})]
|
||||
},
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
await handler.process_output_response(response, guardrail)
|
||||
|
||||
assert _tool_call_ids_names_and_arguments(guardrail.seen_inputs[-1]["tool_calls"]) == [
|
||||
("srvtoolu_01Search", "web_search", '{"query": "x"}')
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_output_guardrail_rewrite_of_a_server_tool_call_lands_in_server_tool_calls_only(self):
|
||||
handler = OpenAIChatCompletionsHandler()
|
||||
weather_args = json.dumps({"city": "Paris", "query": "forecast"})
|
||||
response = ModelResponse(
|
||||
model="claude-sonnet-4-5",
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="tool_calls",
|
||||
index=0,
|
||||
message=Message(
|
||||
content="Looking it up",
|
||||
role="assistant",
|
||||
tool_calls=[
|
||||
ChatCompletionMessageToolCall(
|
||||
id="call_1",
|
||||
type="function",
|
||||
function=Function(name="get_weather", arguments=json.dumps({"city": "Paris"})),
|
||||
)
|
||||
],
|
||||
provider_specific_fields={
|
||||
"server_tool_calls": [
|
||||
_server_tool_call("srvtoolu_01Search", "web_search", {"query": "jane@example.com"})
|
||||
]
|
||||
},
|
||||
),
|
||||
),
|
||||
Choices(
|
||||
finish_reason="tool_calls",
|
||||
index=1,
|
||||
message=Message(
|
||||
content=None,
|
||||
role="assistant",
|
||||
tool_calls=[
|
||||
ChatCompletionMessageToolCall(
|
||||
id="call_2",
|
||||
type="function",
|
||||
function=Function(name="get_weather", arguments=weather_args),
|
||||
)
|
||||
],
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
await handler.process_output_response(response, QueryRedactingGuardrail(guardrail_name="redact"))
|
||||
|
||||
first, second = response.choices
|
||||
assert _tool_call_ids_names_and_arguments(first.message.provider_specific_fields["server_tool_calls"]) == [
|
||||
("srvtoolu_01Search", "web_search", '{"query": "[REDACTED]"}')
|
||||
]
|
||||
assert [(call.id, call.function.arguments) for call in first.message.tool_calls] == [
|
||||
("call_1", '{"city": "Paris"}')
|
||||
]
|
||||
assert [(call.id, call.function.arguments) for call in second.message.tool_calls] == [
|
||||
("call_2", '{"city": "Paris", "query": "[REDACTED]"}')
|
||||
]
|
||||
assert not (second.message.provider_specific_fields or {}).get("server_tool_calls")
|
||||
|
||||
|
||||
class MockPassThroughGuardrail(CustomGuardrail):
|
||||
"""Mock guardrail that passes through without blocking - for testing streaming fallback behavior"""
|
||||
|
|
@ -1462,6 +1584,61 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
|
|||
assert chunks[0].choices[0].delta.content == "hello [MASKED]"
|
||||
assert chunks[1].choices[0].delta.content in (None, "")
|
||||
|
||||
@staticmethod
|
||||
def _ended_server_tool_call_stream_chunks() -> list:
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
def chunk(delta: Delta, finish_reason: Optional[str] = None) -> ModelResponseStream:
|
||||
return ModelResponseStream(
|
||||
id="chatcmpl-123",
|
||||
created=1234567890,
|
||||
model="claude-sonnet-4-5",
|
||||
object="chat.completion.chunk",
|
||||
choices=[StreamingChoices(index=0, delta=delta, finish_reason=finish_reason)],
|
||||
)
|
||||
|
||||
search = {**_server_tool_call("srvtoolu_01Search", "web_search", {"query": "jane@example.com"}), "index": 0}
|
||||
return [
|
||||
chunk(Delta(content="Searching")),
|
||||
chunk(Delta(content=None, provider_specific_fields={"server_tool_calls": [search]})),
|
||||
chunk(Delta(content=None), finish_reason="stop"),
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ended_stream_guardrail_sees_streamed_server_tool_calls(self):
|
||||
handler = OpenAIChatCompletionsHandler()
|
||||
guardrail = self._recording_guardrail()
|
||||
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=self._ended_server_tool_call_stream_chunks(),
|
||||
guardrail_to_apply=guardrail,
|
||||
litellm_logging_obj=None,
|
||||
deliver_ended_stream_rewrites=True,
|
||||
)
|
||||
|
||||
assert _tool_call_ids_names_and_arguments(guardrail.seen_inputs[-1]["tool_calls"]) == [
|
||||
("srvtoolu_01Search", "web_search", '{"query": "jane@example.com"}')
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deliver_ended_stream_rewrites_writes_server_tool_call_arguments_back_into_chunks(self):
|
||||
handler = OpenAIChatCompletionsHandler()
|
||||
chunks = self._ended_server_tool_call_stream_chunks()
|
||||
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
guardrail_to_apply=QueryRedactingGuardrail(guardrail_name="redact"),
|
||||
litellm_logging_obj=None,
|
||||
deliver_ended_stream_rewrites=True,
|
||||
)
|
||||
|
||||
delta = chunks[1].choices[0].delta
|
||||
assert _tool_call_ids_names_and_arguments(delta.provider_specific_fields["server_tool_calls"]) == [
|
||||
("srvtoolu_01Search", "web_search", '{"query": "[REDACTED]"}')
|
||||
]
|
||||
assert delta.tool_calls is None
|
||||
assert chunks[0].choices[0].delta.content == "Searching"
|
||||
|
||||
|
||||
class TestUndecoratedGuardrailIsRecorded:
|
||||
"""LIT-5983 regression: the handler calls apply_guardrail bare, so a custom guardrail
|
||||
|
|
@ -2323,6 +2500,29 @@ class TestStreamingScanKey:
|
|||
assert len(ended_key.tool_calls) == 1 and "run_shell" in ended_key.tool_calls[0]
|
||||
assert ended_key.tool_calls_in_flight is False
|
||||
|
||||
def test_server_tool_calls_hold_the_window_and_enter_the_key_once_the_stream_has_ended(self):
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
handler = OpenAIChatCompletionsHandler()
|
||||
server_chunk = ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(
|
||||
content=None,
|
||||
provider_specific_fields={
|
||||
"server_tool_calls": [_server_tool_call("srvtoolu_01Search", "web_search", {"query": "x"})]
|
||||
},
|
||||
),
|
||||
)
|
||||
]
|
||||
)
|
||||
open_key = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk])
|
||||
ended_key = handler.get_streaming_scan_key([self._chunk("hi"), server_chunk, self._chunk(None, "stop")])
|
||||
assert open_key.tool_calls_in_flight is True
|
||||
assert len(ended_key.tool_calls) == 1 and "web_search" in ended_key.tool_calls[0]
|
||||
assert ended_key != open_key
|
||||
|
||||
def test_text_after_the_first_choice_finishes_still_changes_the_key(self):
|
||||
handler = OpenAIChatCompletionsHandler()
|
||||
first_done = [self._chunk("a", index=0), self._chunk("b", finish_reason="stop", index=0)]
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm.proxy.db.spend_log_tool_index import (
|
|||
)
|
||||
from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import ModelResponse
|
||||
from tests.unit.proxy.db.fake_prisma_engine import engine_call
|
||||
|
||||
|
||||
|
|
@ -204,6 +205,36 @@ class TestResponseToolCallNames:
|
|||
def test_blank_names_are_dropped(self):
|
||||
assert response_tool_call_names(_response_with_tool_calls(" ", "real_tool")) == ("real_tool",)
|
||||
|
||||
@pytest.mark.parametrize("as_object", [False, True], ids=["dict", "model_response"])
|
||||
def test_server_tool_calls_of_every_choice_are_named_before_client_tool_calls(self, as_object: bool):
|
||||
def tool_call(call_id: str, name: str) -> dict[str, object]:
|
||||
return {"id": call_id, "type": "function", "function": {"name": name, "arguments": "{}"}}
|
||||
|
||||
response: dict[str, object] = {
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [tool_call("call_1", "get_weather")],
|
||||
"provider_specific_fields": {"server_tool_calls": [tool_call("srvtoolu_01", "web_search")]},
|
||||
},
|
||||
},
|
||||
{
|
||||
"index": 1,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "done",
|
||||
"provider_specific_fields": {"server_tool_calls": [tool_call("srvtoolu_02", "web_fetch")]},
|
||||
},
|
||||
},
|
||||
]
|
||||
}
|
||||
completion_response = ModelResponse(**response) if as_object else response
|
||||
|
||||
assert response_tool_call_names(completion_response) == ("web_search", "web_fetch", "get_weather")
|
||||
|
||||
def test_responses_api_output_function_calls(self):
|
||||
# Regression: /v1/responses carries invocations in output[] items of
|
||||
# type function_call, not in choices; they must reach the rollup.
|
||||
|
|
|
|||
|
|
@ -27,11 +27,21 @@ from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import (
|
|||
from litellm.types.utils import (
|
||||
ChatCompletionMessageToolCall,
|
||||
Choices,
|
||||
Delta,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
StreamingChoices,
|
||||
)
|
||||
|
||||
|
||||
_SERVER_WEB_SEARCH_CALL: Final = {
|
||||
"id": "srvtoolu_01Search",
|
||||
"type": "function",
|
||||
"function": {"name": "web_search", "arguments": '{"query": "weather in Paris"}'},
|
||||
}
|
||||
_ALLOWED_BASH_CALL: Final = {"id": "toolu_01Bash", "type": "function", "function": {"name": "Bash", "arguments": "{}"}}
|
||||
|
||||
|
||||
class TestToolPermissionGuardrail:
|
||||
"""Test class for Tool Permission Guardrail functionality"""
|
||||
|
||||
|
|
@ -952,6 +962,71 @@ class TestToolPermissionGuardrail:
|
|||
assert "Permission denied" in choice.message.content
|
||||
|
||||
|
||||
def _server_search_response(self) -> ModelResponse:
|
||||
return ModelResponse(
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="tool_calls",
|
||||
message={
|
||||
"content": "Searching.",
|
||||
"tool_calls": [_ALLOWED_BASH_CALL],
|
||||
"provider_specific_fields": {"server_tool_calls": [_SERVER_WEB_SEARCH_CALL]},
|
||||
},
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_denied_server_tool_call_is_blocked(self):
|
||||
with patch.object(self.guardrail, "should_run_guardrail", return_value=True):
|
||||
with pytest.raises(GuardrailRaisedException):
|
||||
await self.guardrail.async_post_call_success_hook(
|
||||
data={}, user_api_key_dict=UserAPIKeyAuth(), response=self._server_search_response()
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_denied_server_tool_call_in_a_chat_stream_is_blocked(self):
|
||||
async def _stream():
|
||||
yield ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(
|
||||
content="Searching.",
|
||||
provider_specific_fields={"server_tool_calls": [{**_SERVER_WEB_SEARCH_CALL, "index": 0}]},
|
||||
),
|
||||
finish_reason="stop",
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
with patch.object(self.guardrail, "should_run_guardrail", return_value=True):
|
||||
with pytest.raises(GuardrailRaisedException):
|
||||
async for _ in self.guardrail.async_post_call_streaming_iterator_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(), response=_stream(), request_data={}
|
||||
):
|
||||
pass
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rewrite_mode_removes_the_denied_server_tool_call(self):
|
||||
rewriting: Final = ToolPermissionGuardrail(
|
||||
guardrail_name="test-tool-permission",
|
||||
rules=self.test_rules,
|
||||
default_action="deny",
|
||||
on_disallowed_action="rewrite",
|
||||
)
|
||||
|
||||
with patch.object(rewriting, "should_run_guardrail", return_value=True):
|
||||
result: Final = await rewriting.async_post_call_success_hook(
|
||||
data={}, user_api_key_dict=UserAPIKeyAuth(), response=self._server_search_response()
|
||||
)
|
||||
|
||||
message: Final = result.choices[0].message
|
||||
assert message.provider_specific_fields["server_tool_calls"] == []
|
||||
assert [tool_call.id for tool_call in message.tool_calls] == [_ALLOWED_BASH_CALL["id"]]
|
||||
assert "Permission denied" in message.content
|
||||
|
||||
|
||||
class TestToolPermissionGuardrailIntegration:
|
||||
"""Integration tests for Tool Permission Guardrail"""
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ from litellm.responses.litellm_completion_transformation.transformation import (
|
|||
LiteLLMCompletionResponsesConfig,
|
||||
)
|
||||
from litellm.types.llms.base import HiddenParams
|
||||
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
|
||||
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse
|
||||
from litellm.types.responses.main import OutputToolSearchCall, build_web_search_call
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionMessageToolCall,
|
||||
|
|
@ -4993,6 +4993,159 @@ def test_function_call_tool_id_falls_back_to_unique_id_for_degenerate_call_id():
|
|||
assert convert(openai)["id"] == "call_tokyo"
|
||||
|
||||
|
||||
_WEATHER_TOOL_USE: Final = {
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01Weather",
|
||||
"name": "get_weather",
|
||||
"input": {"city": "Paris"},
|
||||
}
|
||||
_FETCHED_URL: Final = "https://example.com/"
|
||||
_BLOCKED_URL: Final = "https://blocked.invalid/"
|
||||
_ANTHROPIC_WEB_FETCH_TURN: Final[tuple[dict[str, object], ...]] = (
|
||||
{"type": "server_tool_use", "id": "srvtoolu_01Fetch", "name": "web_fetch", "input": {"url": _FETCHED_URL}},
|
||||
{
|
||||
"type": "web_fetch_tool_result",
|
||||
"tool_use_id": "srvtoolu_01Fetch",
|
||||
"content": {
|
||||
"type": "web_fetch_result",
|
||||
"url": _FETCHED_URL,
|
||||
"content": {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "Example"}},
|
||||
},
|
||||
},
|
||||
{"type": "server_tool_use", "id": "srvtoolu_02Fetch", "name": "web_fetch", "input": {"url": _BLOCKED_URL}},
|
||||
{
|
||||
"type": "web_fetch_tool_result",
|
||||
"tool_use_id": "srvtoolu_02Fetch",
|
||||
"content": {"type": "web_fetch_tool_result_error", "error_code": "url_not_allowed"},
|
||||
},
|
||||
{"type": "text", "text": "The page is titled Example."},
|
||||
_WEATHER_TOOL_USE,
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_turn_on_responses(content: Sequence[Mapping[str, object]]) -> ResponsesAPIResponse:
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
|
||||
anthropic_response: Final = {
|
||||
"id": "msg_01Fetch",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5-5",
|
||||
"content": list(content),
|
||||
"stop_reason": "tool_use",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 100, "output_tokens": 50},
|
||||
}
|
||||
raw_response: Final = MagicMock(spec=httpx.Response)
|
||||
raw_response.json.return_value = anthropic_response
|
||||
raw_response.status_code = 200
|
||||
raw_response.headers = {}
|
||||
chat_response: Final = AnthropicConfig().transform_parsed_response(
|
||||
completion_response=anthropic_response,
|
||||
raw_response=raw_response,
|
||||
model_response=ModelResponse(),
|
||||
json_mode=False,
|
||||
prefix_prompt=None,
|
||||
)
|
||||
return LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(
|
||||
request_input="Use the web and check the weather in Paris.",
|
||||
responses_api_request={
|
||||
"tools": [
|
||||
{"type": "web_search_20250305", "name": "web_search"},
|
||||
{"type": "web_fetch_20250910", "name": "web_fetch"},
|
||||
{"type": "function", "name": "get_weather", "parameters": {"type": "object"}},
|
||||
]
|
||||
},
|
||||
chat_completion_response=chat_response,
|
||||
)
|
||||
|
||||
|
||||
def test_anthropic_web_fetch_is_a_hosted_open_page_call_on_responses() -> None:
|
||||
response: Final = _anthropic_turn_on_responses(_ANTHROPIC_WEB_FETCH_TURN)
|
||||
|
||||
assert [
|
||||
(item.id, item.status, item.action.model_dump()) for item in response.output if item.type == "web_search_call"
|
||||
] == [
|
||||
("ws_srvtoolu_01Fetch", "completed", {"type": "open_page", "url": _FETCHED_URL}),
|
||||
("ws_srvtoolu_02Fetch", "failed", {"type": "open_page", "url": _BLOCKED_URL}),
|
||||
]
|
||||
assert [item.name for item in response.output if item.type == "function_call"] == ["get_weather"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("server_tool_use", "expected_action"),
|
||||
[
|
||||
(
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_01Pending",
|
||||
"name": "web_fetch",
|
||||
"input": {"url": _FETCHED_URL},
|
||||
},
|
||||
{"type": "open_page", "url": _FETCHED_URL},
|
||||
),
|
||||
(
|
||||
{"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_search", "input": {"query": "python"}},
|
||||
{"type": "search", "query": "python"},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_anthropic_server_web_call_without_a_result_is_an_in_progress_hosted_call_on_responses(
|
||||
server_tool_use: Mapping[str, object], expected_action: Mapping[str, object]
|
||||
) -> None:
|
||||
response: Final = _anthropic_turn_on_responses((server_tool_use, _WEATHER_TOOL_USE))
|
||||
|
||||
web_calls: Final = [item for item in response.output if item.type == "web_search_call"]
|
||||
assert [(item.id, item.status) for item in web_calls] == [("ws_srvtoolu_01Pending", "in_progress")]
|
||||
assert expected_action.items() <= web_calls[0].action.model_dump().items()
|
||||
assert [item.name for item in response.output if item.type == "function_call"] == ["get_weather"]
|
||||
|
||||
|
||||
def test_anthropic_code_execution_is_never_a_client_function_call_on_responses() -> None:
|
||||
response: Final = _anthropic_turn_on_responses(
|
||||
(
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_01Bash",
|
||||
"name": "bash_code_execution",
|
||||
"input": {"command": "ls"},
|
||||
},
|
||||
{
|
||||
"type": "bash_code_execution_tool_result",
|
||||
"tool_use_id": "srvtoolu_01Bash",
|
||||
"content": {
|
||||
"type": "bash_code_execution_result",
|
||||
"stdout": "notes.txt\n",
|
||||
"stderr": "",
|
||||
"return_code": 0,
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_02Edit",
|
||||
"name": "text_editor_code_execution",
|
||||
"input": {"command": "view", "path": "notes.txt"},
|
||||
},
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_03Bash",
|
||||
"name": "bash_code_execution",
|
||||
"input": {"command": "pwd"},
|
||||
},
|
||||
_WEATHER_TOOL_USE,
|
||||
)
|
||||
)
|
||||
|
||||
assert [(item.type, getattr(item, "call_id", item.id)) for item in response.output if item.type != "message"] == [
|
||||
("code_interpreter_call", "srvtoolu_01Bash"),
|
||||
("function_call", "toolu_01Weather"),
|
||||
]
|
||||
|
||||
|
||||
class TestHostedWebSearchReplay:
|
||||
def test_emitted_hosted_search_output_round_trips_with_client_tool_result(self) -> None:
|
||||
search_result: Final = {
|
||||
|
|
@ -5063,6 +5216,75 @@ class TestHostedWebSearchReplay:
|
|||
assert messages[2]["content"] == "Paris is sunny."
|
||||
assert history == original
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"leading_text", ((), ({"type": "text", "text": "Let me look that up."},)), ids=("alone", "after_text")
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("server_tool_use", "expected_input"),
|
||||
(
|
||||
(
|
||||
{"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_fetch", "input": {"url": _FETCHED_URL}},
|
||||
{"url": _FETCHED_URL},
|
||||
),
|
||||
(
|
||||
{"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_search", "input": {"query": "python"}},
|
||||
{"query": "python"},
|
||||
),
|
||||
),
|
||||
ids=("web_fetch", "web_search"),
|
||||
)
|
||||
def test_pending_claude_web_call_replays_to_anthropic_as_its_own_server_tool_use(
|
||||
self,
|
||||
leading_text: tuple[Mapping[str, object], ...],
|
||||
server_tool_use: Mapping[str, object],
|
||||
expected_input: Mapping[str, object],
|
||||
) -> None:
|
||||
response: Final = _anthropic_turn_on_responses((*leading_text, server_tool_use, _WEATHER_TOOL_USE))
|
||||
history: Final = [
|
||||
{"role": "user", "content": "Use the web and check the weather in Paris."},
|
||||
*(item.model_dump(exclude_none=True) for item in response.output),
|
||||
{"type": "function_call_output", "call_id": "toolu_01Weather", "output": "Paris is sunny."},
|
||||
]
|
||||
|
||||
anthropic_turns: Final = anthropic_messages_pt(
|
||||
messages=LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=history, responses_api_request={}
|
||||
),
|
||||
model="claude-sonnet-5-5",
|
||||
llm_provider="anthropic",
|
||||
)
|
||||
|
||||
assert [turn["role"] for turn in anthropic_turns] == ["user", "assistant", "user"]
|
||||
assert [block for block in anthropic_turns[1]["content"] if block["type"] != "text"] == [
|
||||
{"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": server_tool_use["name"], "input": expected_input},
|
||||
{"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}},
|
||||
]
|
||||
assert [(block["type"], block["tool_use_id"]) for block in anthropic_turns[2]["content"]] == [
|
||||
("tool_result", "toolu_01Weather")
|
||||
]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("call_id", "status"),
|
||||
(("ws_srvtoolu_01Fetch", "completed"), ("ws_68a1b2c3d4", "in_progress"), ("srvtoolu_01Fetch", "in_progress")),
|
||||
ids=("completed-claude-call", "pending-non-claude-call", "pending-unprefixed-id"),
|
||||
)
|
||||
def test_web_call_that_is_not_a_pending_claude_call_replays_as_text_only(
|
||||
self, call_id: str, status: Literal["completed", "in_progress"]
|
||||
) -> None:
|
||||
search: Final = ResponseFunctionWebSearch(
|
||||
id=call_id, type="web_search_call", status=status, action=ActionOpenPage(type="open_page", url=_FETCHED_URL)
|
||||
)
|
||||
|
||||
messages: Final = (
|
||||
LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message(
|
||||
input_item=search.model_dump(exclude_none=True)
|
||||
)
|
||||
)
|
||||
|
||||
assert messages == [
|
||||
{"role": "assistant", "content": "Hosted web search: " + search.model_dump_json(exclude_none=True)}
|
||||
]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"action",
|
||||
(
|
||||
|
|
|
|||
|
|
@ -356,6 +356,198 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to
|
|||
)
|
||||
|
||||
|
||||
_STREAMED_WEB_FETCH_CALL: Final[tuple[dict[str, object], ...]] = (
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_01Fetch",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [],
|
||||
"usage": {"input_tokens": 10, "output_tokens": 1},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 0,
|
||||
"content_block": {"type": "server_tool_use", "id": "srvtoolu_01Fetch", "name": "web_fetch"},
|
||||
},
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": 0,
|
||||
"delta": {"type": "input_json_delta", "partial_json": '{"url": "https://example.com/"}'},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
)
|
||||
_STREAMED_WEB_FETCH_RESULT: Final[tuple[dict[str, object], ...]] = (
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 1,
|
||||
"content_block": {
|
||||
"type": "web_fetch_tool_result",
|
||||
"tool_use_id": "srvtoolu_01Fetch",
|
||||
"content": {"type": "web_fetch_result", "url": "https://example.com/"},
|
||||
},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 1},
|
||||
)
|
||||
|
||||
|
||||
def _streamed_text_then_weather_call(first_index: int) -> tuple[dict[str, object], ...]:
|
||||
return (
|
||||
{"type": "content_block_start", "index": first_index, "content_block": {"type": "text", "text": ""}},
|
||||
{"type": "content_block_delta", "index": first_index, "delta": {"type": "text_delta", "text": "Fetched."}},
|
||||
{"type": "content_block_stop", "index": first_index},
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": first_index + 1,
|
||||
"content_block": {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {}},
|
||||
},
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": first_index + 1,
|
||||
"delta": {"type": "input_json_delta", "partial_json": '{"city": "Paris"}'},
|
||||
},
|
||||
{"type": "content_block_stop", "index": first_index + 1},
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "tool_use", "stop_sequence": None},
|
||||
"usage": {"output_tokens": 20},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_web_fetch_stream_events(fetch_finished: bool) -> tuple[dict[str, object], ...]:
|
||||
if fetch_finished:
|
||||
return (*_STREAMED_WEB_FETCH_CALL, *_STREAMED_WEB_FETCH_RESULT, *_streamed_text_then_weather_call(2))
|
||||
return (*_STREAMED_WEB_FETCH_CALL, *_streamed_text_then_weather_call(1))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
("fetch_finished", "expected_status", "expected_search_events"),
|
||||
[
|
||||
pytest.param(
|
||||
True,
|
||||
"completed",
|
||||
[
|
||||
ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
|
||||
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS,
|
||||
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING,
|
||||
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED,
|
||||
ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
],
|
||||
id="fetch_finished_in_this_turn",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
"in_progress",
|
||||
[
|
||||
ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
|
||||
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS,
|
||||
ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING,
|
||||
ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
],
|
||||
id="fetch_still_waiting_on_its_result",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_anthropic_web_fetch_is_a_hosted_open_page_call(
|
||||
sync_mode: bool,
|
||||
fetch_finished: bool,
|
||||
expected_status: str,
|
||||
expected_search_events: list[ResponsesAPIStreamEvents],
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
|
||||
|
||||
parser: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=sync_mode, json_mode=False)
|
||||
chunks: Final = [parser.chunk_parser(event) for event in _anthropic_web_fetch_stream_events(fetch_finished)]
|
||||
iterator: Final = LiteLLMCompletionStreamingIterator(
|
||||
model="claude-sonnet-5-5",
|
||||
litellm_custom_stream_wrapper=_FakeStreamWrapper(chunks),
|
||||
request_input="Fetch example.com and check the weather in Paris.",
|
||||
responses_api_request={
|
||||
"tools": [
|
||||
{"type": "web_fetch_20250910", "name": "web_fetch"},
|
||||
{"type": "function", "name": "get_weather", "parameters": {"type": "object"}},
|
||||
]
|
||||
},
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
|
||||
events: Final = await _collect_events(iterator, sync_mode)
|
||||
|
||||
completed: Final = next(
|
||||
event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
|
||||
)
|
||||
assert [
|
||||
(item.id, item.status, item.action.model_dump())
|
||||
for item in completed.response.output
|
||||
if item.type == "web_search_call"
|
||||
] == [("ws_srvtoolu_01Fetch", expected_status, {"type": "open_page", "url": "https://example.com/"})]
|
||||
assert [
|
||||
event.type
|
||||
for event in events
|
||||
if getattr(event, "item_id", getattr(getattr(event, "item", None), "id", None)) == "ws_srvtoolu_01Fetch"
|
||||
] == expected_search_events
|
||||
assert [item.name for item in completed.response.output if item.type == "function_call"] == ["get_weather"]
|
||||
assert not any(
|
||||
event.item.name == "web_fetch"
|
||||
for event in events
|
||||
if getattr(event, "type", None) == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED
|
||||
and event.item.type == "function_call"
|
||||
)
|
||||
|
||||
|
||||
_STREAMED_PENDING_CODE_EXECUTION: Final[tuple[dict[str, object], ...]] = (
|
||||
_STREAMED_WEB_FETCH_CALL[0],
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 0,
|
||||
"content_block": {"type": "server_tool_use", "id": "srvtoolu_01Bash", "name": "bash_code_execution"},
|
||||
},
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": 0,
|
||||
"delta": {"type": "input_json_delta", "partial_json": '{"command": "ls"}'},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
*_streamed_text_then_weather_call(1),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_anthropic_code_execution_is_never_a_client_function_call(sync_mode: bool) -> None:
|
||||
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
|
||||
|
||||
parser: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=sync_mode, json_mode=False)
|
||||
iterator: Final = LiteLLMCompletionStreamingIterator(
|
||||
model="claude-sonnet-5-5",
|
||||
litellm_custom_stream_wrapper=_FakeStreamWrapper(
|
||||
[parser.chunk_parser(event) for event in _STREAMED_PENDING_CODE_EXECUTION]
|
||||
),
|
||||
request_input="List the files, then check the weather in Paris.",
|
||||
responses_api_request={
|
||||
"tools": [
|
||||
{"type": "code_execution_20250825", "name": "code_execution"},
|
||||
{"type": "function", "name": "get_weather", "parameters": {"type": "object"}},
|
||||
]
|
||||
},
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
|
||||
events: Final = await _collect_events(iterator, sync_mode)
|
||||
|
||||
item_events: Final = [event for event in events if getattr(event, "item", None) is not None]
|
||||
assert {event.item.call_id for event in item_events if event.item.type == "function_call"} == {"toolu_01Weather"}
|
||||
completed: Final = next(
|
||||
event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
|
||||
)
|
||||
assert [item.call_id for item in completed.response.output if item.type == "function_call"] == ["toolu_01Weather"]
|
||||
|
||||
|
||||
def test_tool_calls_present_only_in_final_response_are_emitted_before_completed():
|
||||
iterator = LiteLLMCompletionStreamingIterator(
|
||||
model="test-model",
|
||||
|
|
|
|||
|
|
@ -131,6 +131,52 @@ describe("ToolsSection utils", () => {
|
|||
expect(result[1].callData).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should mark Claude server tools as called from provider_specific_fields.server_tool_calls", () => {
|
||||
const log: Partial<LogEntry> = {
|
||||
request_id: "test-server-tools",
|
||||
proxy_server_request: {
|
||||
tools: [
|
||||
{ type: "web_search_20250305", name: "web_search", max_uses: 3 },
|
||||
{ type: "function", function: { name: "get_weather", description: "Get weather" } },
|
||||
{ type: "function", function: { name: "send_email", description: "Send email" } },
|
||||
],
|
||||
},
|
||||
response: {
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
tool_calls: [
|
||||
{
|
||||
id: "toolu_01Weather",
|
||||
type: "function",
|
||||
function: { name: "get_weather", arguments: '{"location": "Paris"}' },
|
||||
},
|
||||
],
|
||||
provider_specific_fields: {
|
||||
server_tool_calls: [
|
||||
{
|
||||
id: "srvtoolu_01Search",
|
||||
type: "function",
|
||||
function: { name: "web_search", arguments: '{"query": "paris weather"}' },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
const result = parseToolsFromLog(log as LogEntry);
|
||||
|
||||
expect(result.map((tool) => [tool.name, tool.called, tool.callData?.id])).toEqual([
|
||||
["web_search", true, "srvtoolu_01Search"],
|
||||
["get_weather", true, "toolu_01Weather"],
|
||||
["send_email", false, undefined],
|
||||
]);
|
||||
expect(result[0].callData?.arguments).toEqual({ query: "paris weather" });
|
||||
});
|
||||
|
||||
it("should handle string format request and response", () => {
|
||||
const log: Partial<LogEntry> = {
|
||||
request_id: "test-5",
|
||||
|
|
|
|||
|
|
@ -42,6 +42,24 @@ function extractToolsFromRequest(log: LogEntry): ToolDefinition[] {
|
|||
return [];
|
||||
}
|
||||
|
||||
interface ChatMessageToolCallFields {
|
||||
tool_calls?: unknown;
|
||||
provider_specific_fields?: { server_tool_calls?: unknown } | null;
|
||||
}
|
||||
|
||||
function toolCallList(value: unknown): ToolCall[] {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function extractChatMessageToolCalls(message: ChatMessageToolCallFields | undefined): ToolCall[] | null {
|
||||
const serverToolCalls = message?.provider_specific_fields?.server_tool_calls;
|
||||
const clientToolCalls = message?.tool_calls;
|
||||
if (!Array.isArray(serverToolCalls) && !Array.isArray(clientToolCalls)) {
|
||||
return null;
|
||||
}
|
||||
return [...toolCallList(serverToolCalls), ...toolCallList(clientToolCalls)];
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract tool calls from response data
|
||||
*/
|
||||
|
|
@ -53,10 +71,9 @@ function extractToolCallsFromResponse(log: LogEntry): ToolCall[] {
|
|||
// OpenAI format: response.choices[0].message.tool_calls
|
||||
const choices = responseData.choices;
|
||||
if (Array.isArray(choices) && choices.length > 0) {
|
||||
const firstChoice = choices[0];
|
||||
const message = firstChoice.message;
|
||||
if (message && Array.isArray(message.tool_calls)) {
|
||||
return message.tool_calls;
|
||||
const chatToolCalls = extractChatMessageToolCalls(choices[0].message);
|
||||
if (chatToolCalls) {
|
||||
return chatToolCalls;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue