diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index d1323b1a2bf..093dffccac0 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -197,12 +197,13 @@ class OpenAIResponsesHandler(BaseTranslation): return data def extract_request_tool_names(self, data: dict) -> List[str]: - """Extract tool names from Responses API request (tools[].name for function, tools[].server_label for mcp).""" + """Extract tool names from Responses API request (tools[].name for function + and custom, tools[].server_label for mcp).""" names: List[str] = [] for tool in data.get("tools") or []: if not isinstance(tool, dict): continue - if tool.get("type") == "function" and tool.get("name"): + if tool.get("type") in ("function", "custom") and tool.get("name"): names.append(str(tool["name"])) elif tool.get("type") == "mcp" and tool.get("server_label"): names.append(str(tool["server_label"])) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index c4be2187292..abaf82fb661 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -91,7 +91,9 @@ _CLIENT_DISCONNECTED_ERROR_INFORMATION: StandardLoggingPayloadErrorInformation = } -def _apply_client_disconnect_metadata(target_metadata: dict[str, object]) -> None: +def _apply_client_disconnect_metadata(target_metadata: Optional[dict[str, object]]) -> None: + if target_metadata is None: + return target_metadata["client_disconnected"] = True target_metadata["error_information"] = dict(_CLIENT_DISCONNECTED_ERROR_INFORMATION) @@ -114,12 +116,33 @@ async def _record_streaming_client_disconnect_if_needed( logging_obj = request_data.get("litellm_logging_obj") if logging_obj is not None: litellm_params = logging_obj.model_call_details.setdefault("litellm_params", {}) - _apply_client_disconnect_metadata(litellm_params.setdefault("metadata", {})) - _apply_client_disconnect_metadata(logging_obj.model_call_details.setdefault("metadata", {})) + _lp_metadata = litellm_params.get("metadata") + if _lp_metadata is None: + _lp_metadata = {} + litellm_params["metadata"] = _lp_metadata + _apply_client_disconnect_metadata(_lp_metadata) - _apply_client_disconnect_metadata(request_data.setdefault("metadata", {})) - litellm_params = request_data.setdefault("litellm_params", {}) - _apply_client_disconnect_metadata(litellm_params.setdefault("metadata", {})) + _mcd_metadata = logging_obj.model_call_details.get("metadata") + if _mcd_metadata is None: + _mcd_metadata = {} + logging_obj.model_call_details["metadata"] = _mcd_metadata + _apply_client_disconnect_metadata(_mcd_metadata) + + _rd_metadata = request_data.get("metadata") + if _rd_metadata is None: + _rd_metadata = {} + request_data["metadata"] = _rd_metadata + _apply_client_disconnect_metadata(_rd_metadata) + + _rd_litellm_params = request_data.get("litellm_params") + if _rd_litellm_params is None: + _rd_litellm_params = {} + request_data["litellm_params"] = _rd_litellm_params + _rd_lp_metadata = _rd_litellm_params.get("metadata") + if _rd_lp_metadata is None: + _rd_lp_metadata = {} + _rd_litellm_params["metadata"] = _rd_lp_metadata + _apply_client_disconnect_metadata(_rd_lp_metadata) verbose_proxy_logger.debug( "Recorded streaming client disconnect with error_code=499 for litellm_call_id=%s", diff --git a/litellm/responses/litellm_completion_transformation/custom_tools.py b/litellm/responses/litellm_completion_transformation/custom_tools.py new file mode 100644 index 00000000000..2417bf5cf2e --- /dev/null +++ b/litellm/responses/litellm_completion_transformation/custom_tools.py @@ -0,0 +1,163 @@ +""" +Utilities for handling OpenAI Responses API 'custom' tools (freeform/grammar tools) +when bridging to Chat Completions providers. + +Custom tools are defined with ``type: "custom"`` and a grammar/format specification. +Since most Chat Completions providers only support standard ``function`` tools, +the bridge converts them to ``function`` tools with a single ``content`` string +parameter. When the model responds with a ``function_call`` for such a tool, this +module converts it back to the ``custom_tool_call`` format expected by clients like +Codex CLI. + +The forward direction (custom -> function) and reverse direction (function_call -> +custom_tool_call) are both handled here so future custom tool types can be added by +extending this module without touching the streaming iterator or transformation +logic. +""" + +import json +from collections.abc import Mapping +from typing import Any + +from pydantic import BaseModel, TypeAdapter, ValidationError + +from litellm.types.llms.openai import ( + ChatCompletionToolParam, + ChatCompletionToolParamFunctionChunk, +) + +_MAX_ARGUMENTS_LEN = 1_000_000 + + +def extract_custom_tool_names(tools: list[Any] | None) -> set[str]: + """Extract names of tools originally defined as ``type: "custom"``.""" + if not tools: + return set() + names: set[str] = set() + for tool in tools: + if isinstance(tool, dict) and tool.get("type") == "custom" and "name" in tool: + names.add(tool["name"]) + return names + + +def is_custom_tool_call(tool_name: str, custom_tool_names: set[str]) -> bool: + """Check if a tool call name corresponds to a custom tool.""" + return tool_name in custom_tool_names + + +def unwrap_custom_tool_arguments(arguments: str) -> str: + """Extract the raw content string from JSON-wrapped arguments. + + The bridge converts custom tools to function tools with schema + ``{"properties": {"content": {"type": "string"}}}``, so the model returns + arguments like ``{"content": "*** Begin Patch\\n..."}``. This function + extracts just the content string. If the arguments are not valid JSON or do + not contain a ``content`` key, the original string is returned unchanged. + """ + if not arguments: + return "" + if len(arguments) > _MAX_ARGUMENTS_LEN: + return arguments + try: + parsed = json.loads(arguments) + if isinstance(parsed, dict) and "content" in parsed: + return str(parsed["content"]) + except (json.JSONDecodeError, TypeError, ValueError): + pass + return arguments + + +def build_tool_call_item_kwargs( + call_id: str, + name: str, + arguments_or_input: str, + status: str, + custom_tool_names: set[str], +) -> dict[str, Any]: + """Build kwargs for an output item dict that is either a ``function_call`` + or a ``custom_tool_call`` depending on whether *name* is in + *custom_tool_names*. + + For custom tools the ``arguments`` JSON is unwrapped into the ``input`` + field. For regular function tools the raw ``arguments`` string is kept. + + This centralises the branching logic so the streaming iterator and the + non-streaming transformation share a single code path. + """ + custom = is_custom_tool_call(name, custom_tool_names) + item_type = "custom_tool_call" if custom else "function_call" + kwargs: dict[str, Any] = { + "type": item_type, + "id": call_id, + "call_id": call_id, + "name": name, + "status": status, + } + if custom: + if status == "completed": + kwargs["input"] = unwrap_custom_tool_arguments(arguments_or_input) + else: + kwargs["input"] = "" + else: + kwargs["arguments"] = arguments_or_input + return kwargs + + +class _CustomToolFormat(BaseModel): + syntax: str = "" + definition: str = "" + + +_ALLOWED_CALLERS_ADAPTER = TypeAdapter(list[str] | None) + + +def _validated_allowed_callers(value: object) -> list[str] | None: + try: + return _ALLOWED_CALLERS_ADAPTER.validate_python(value, strict=True) + except ValidationError as exc: + raise ValueError("allowed_callers must be a list of strings") from exc + + +def _grammar_suffix(fmt: object) -> str: + try: + parsed = _CustomToolFormat.model_validate(fmt) + except ValidationError: + return "" + if not parsed.definition: + return "" + return f"\n\nFormat:\n```{parsed.syntax}\n{parsed.definition}\n```" + + +def convert_custom_tool_to_function_tool(tool: Mapping[str, object]) -> ChatCompletionToolParam | None: + """Convert a Responses API ``custom`` tool to a Chat Completions ``function`` + tool. + + The grammar definition is embedded in the description so the model can + produce correctly-formatted output. Returns ``None`` if the tool is not a + custom tool. Raises ``ValueError`` if ``allowed_callers`` is not a list of + strings. + """ + if tool.get("type") != "custom": + return None + raw_name = tool.get("name") + name = raw_name if isinstance(raw_name, str) else "" + raw_description = tool.get("description") + description = (raw_description if isinstance(raw_description, str) else "") + _grammar_suffix(tool.get("format")) + allowed_callers = _validated_allowed_callers(tool.get("allowed_callers")) + function_chunk = ChatCompletionToolParamFunctionChunk( + name=name, + description=description, + parameters={ + "type": "object", + "properties": { + "content": { + "type": "string", + "description": f"The {name} content following the specified format", + } + }, + "required": ["content"], + }, + ) + if allowed_callers is None: + return ChatCompletionToolParam(type="function", function=function_chunk) + return ChatCompletionToolParam(type="function", function=function_chunk, allowed_callers=allowed_callers) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index b1198780bac..cf69654d15d 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1,9 +1,13 @@ import time import uuid -from typing import Any, Dict, List, Optional, Union, cast +from typing import Any, cast import litellm from litellm.main import stream_chunk_builder +from litellm.responses.litellm_completion_transformation.custom_tools import ( + build_tool_call_item_kwargs, + extract_custom_tool_names, +) from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) @@ -53,20 +57,20 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self, model: str, litellm_custom_stream_wrapper: litellm.CustomStreamWrapper, - request_input: Union[str, ResponseInputParam], + request_input: str | ResponseInputParam, responses_api_request: ResponsesAPIOptionalRequestParams, - custom_llm_provider: Optional[str] = None, - litellm_metadata: Optional[dict] = None, + custom_llm_provider: str | None = None, + litellm_metadata: dict | None = None, ): self.model: str = model self.litellm_custom_stream_wrapper: litellm.CustomStreamWrapper = litellm_custom_stream_wrapper - self.request_input: Union[str, ResponseInputParam] = request_input + self.request_input: str | ResponseInputParam = request_input self.responses_api_request: ResponsesAPIOptionalRequestParams = responses_api_request - self.custom_llm_provider: Optional[str] = custom_llm_provider - self.litellm_metadata: Optional[dict] = litellm_metadata or {} + self.custom_llm_provider: str | None = custom_llm_provider + self.litellm_metadata: dict | None = litellm_metadata or {} # Store lightweight dict snapshots for stream_chunk_builder to reduce # repeated Pydantic attribute access in end-of-stream assembly. - self.collected_chat_completion_chunks: List[Dict[str, Any]] = [] + self.collected_chat_completion_chunks: list[dict[str, Any]] = [] self.finished: bool = False self.litellm_logging_obj = litellm_custom_stream_wrapper.logging_obj self.sent_response_created_event: bool = False @@ -77,11 +81,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self.sent_output_content_part_done_event: bool = False self.sent_output_item_done_event: bool = False self.sent_annotation_events: bool = False - self.litellm_model_response: Optional[Union[ModelResponse, TextCompletionResponse]] = None + self.litellm_model_response: ModelResponse | TextCompletionResponse | None = None self.final_text: str = "" - self._cached_item_id: Optional[str] = None - self._cached_response_id: Optional[str] = None - self._pending_tool_events: List[BaseLiteLLMOpenAIResponseObject] = [] + self._cached_item_id: str | None = None + self._cached_response_id: str | None = None + self._pending_tool_events: list[BaseLiteLLMOpenAIResponseObject] = [] self._tool_output_index_by_call_id: dict[str, int] = {} self._tool_args_by_call_id: dict[str, str] = {} self._tool_call_id_by_index: dict[int, str] = {} @@ -89,17 +93,18 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._next_tool_output_index: int = 1 # output_index=0 reserved for the message item self._final_tool_events_queued: bool = False self._sequence_number: int = 0 - self._cached_reasoning_item_id: Optional[str] = None + self._cached_reasoning_item_id: str | None = None self._sent_reasoning_summary_text_done_event: bool = False self._sent_reasoning_summary_part_done_event: bool = False self._reasoning_summary_text: str = "" # -- GENERIC RESPONSE-EVENTS PENDING QUEUE as required by fix -- - self._pending_response_events: List[BaseLiteLLMOpenAIResponseObject] = [] + self._pending_response_events: list[BaseLiteLLMOpenAIResponseObject] = [] self._reasoning_active = False self._reasoning_done_emitted = False - self._reasoning_item_id: Optional[str] = None - self._accumulated_reasoning_content_parts: List[str] = [] - self._accumulated_provider_specific_fields: Dict[str, Any] = {} + self._reasoning_item_id: str | None = None + self._accumulated_reasoning_content_parts: list[str] = [] + self._accumulated_provider_specific_fields: dict[str, Any] = {} + self._custom_tool_names: set[str] = extract_custom_tool_names(self.responses_api_request.get("tools")) def _get_or_assign_tool_output_index(self, call_id: str) -> int: existing = self._tool_output_index_by_call_id.get(call_id) @@ -110,7 +115,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._tool_output_index_by_call_id[call_id] = idx return idx - def _normalize_tool_call_index(self, tool_call: object) -> Optional[int]: + def _normalize_tool_call_index(self, tool_call: object) -> int | None: idx_raw = tool_call.get("index") if isinstance(tool_call, dict) else getattr(tool_call, "index", None) if idx_raw is None: return None @@ -183,19 +188,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if call_id not in self._tool_args_by_call_id: self._tool_args_by_call_id[call_id] = "" self._sequence_number += 1 + item_kwargs = build_tool_call_item_kwargs(call_id, fn_name, "", "in_progress", self._custom_tool_names) event = OutputItemAddedEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, output_index=output_index, - item=BaseLiteLLMOpenAIResponseObject( - **{ - "type": "function_call", - "id": call_id, - "call_id": call_id, - "name": fn_name, - "arguments": "", - "status": "in_progress", - } - ), + item=BaseLiteLLMOpenAIResponseObject(**item_kwargs), ) event.__dict__["sequence_number"] = self._sequence_number self._pending_tool_events.append(event) @@ -260,19 +257,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if is_new_tool_call: self._tool_args_by_call_id[call_id] = "" self._sequence_number += 1 + item_kwargs = build_tool_call_item_kwargs(call_id, fn_name, "", "in_progress", self._custom_tool_names) event = OutputItemAddedEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, output_index=output_index, - item=BaseLiteLLMOpenAIResponseObject( - **{ - "type": "function_call", - "id": call_id, - "call_id": call_id, - "name": fn_name, - "arguments": "", - "status": "in_progress", - } - ), + item=BaseLiteLLMOpenAIResponseObject(**item_kwargs), ) event.__dict__["sequence_number"] = self._sequence_number self._pending_tool_events.append(event) @@ -310,20 +299,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._pending_tool_events.append(done_event) self._sequence_number += 1 + item_kwargs = build_tool_call_item_kwargs( + call_id, fn_name, final_args, "completed", self._custom_tool_names + ) item_done_event = OutputItemDoneEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, output_index=output_index, sequence_number=self._sequence_number, - item=BaseLiteLLMOpenAIResponseObject( - **{ - "type": "function_call", - "id": call_id, - "call_id": call_id, - "name": fn_name, - "arguments": final_args, - "status": "completed", - } - ), + item=BaseLiteLLMOpenAIResponseObject(**item_kwargs), ) self._pending_tool_events.append(item_done_event) @@ -449,9 +432,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): for key, val in src.items(): self._accumulated_provider_specific_fields[key] = val - def create_litellm_model_response(self) -> Optional[ModelResponse]: + def create_litellm_model_response(self) -> ModelResponse | None: response = cast( - Optional[ModelResponse], + ModelResponse | None, stream_chunk_builder( chunks=self.collected_chat_completion_chunks, logging_obj=self.litellm_logging_obj, @@ -468,7 +451,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): @staticmethod def _snapshot_chunk_for_stream_chunk_builder( chunk: ModelResponseStream, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: """ Convert a streaming chunk into a plain dict for end-of-stream assembly. Keep _hidden_params so downstream usage/header behavior is preserved. @@ -564,7 +547,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): reasoning_content = getattr(litellm_complete_object.choices[0].message, "reasoning_content", "") or "" # type: ignore annotations = getattr(litellm_complete_object.choices[0].message, "annotations", None) # type: ignore - part: Optional[PART_UNION_TYPES] = None + part: PART_UNION_TYPES | None = None if reasoning_content: part = ContentPartDonePartReasoningText( type="reasoning_text", @@ -671,7 +654,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def return_default_done_events( self, litellm_complete_object: ModelResponse - ) -> Optional[BaseLiteLLMOpenAIResponseObject]: + ) -> BaseLiteLLMOpenAIResponseObject | None: if self.sent_output_text_done_event is False: self.sent_output_text_done_event = True return self.create_output_text_done_event(litellm_complete_object) @@ -685,7 +668,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def return_default_initial_events( self, - ) -> Optional[BaseLiteLLMOpenAIResponseObject]: + ) -> BaseLiteLLMOpenAIResponseObject | None: if self.sent_response_created_event is False: self.sent_response_created_event = True return self.create_response_created_event() @@ -725,6 +708,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self.finished = self.is_stream_finished() response_completed_event = self._emit_response_completed_event(self.litellm_model_response) if response_completed_event: + # Latch so wrappers (FallbackResponsesStreamWrapper) + proxy + # container-ownership hook can read completed_response. + self.completed_response = response_completed_event return response_completed_event else: if sync_mode: @@ -800,11 +786,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): async def __anext__( self, - ) -> Union[ - ResponsesAPIStreamingResponse, - ResponseCompletedEvent, - BaseLiteLLMOpenAIResponseObject, - ]: + ) -> ResponsesAPIStreamingResponse | ResponseCompletedEvent | BaseLiteLLMOpenAIResponseObject: try: while True: if self.finished is True: @@ -906,11 +888,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def __next__( self, - ) -> Union[ - ResponsesAPIStreamingResponse, - ResponseCompletedEvent, - BaseLiteLLMOpenAIResponseObject, - ]: + ) -> ResponsesAPIStreamingResponse | ResponseCompletedEvent | BaseLiteLLMOpenAIResponseObject: try: while True: if self.finished is True: @@ -961,7 +939,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def _transform_chat_completion_chunk_to_response_api_chunk( self, chunk: ModelResponseStream - ) -> Optional[ResponsesAPIStreamingResponse]: + ) -> ResponsesAPIStreamingResponse | None: """ Transform a chat completion chunk to a response API chunk. @@ -1047,7 +1025,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return None - def _get_delta_string_from_streaming_choices(self, choices: List[StreamingChoices]) -> str: + def _get_delta_string_from_streaming_choices(self, choices: list[StreamingChoices]) -> str: """ Get the delta string from the streaming choices @@ -1059,7 +1037,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): chat_completion_delta: ChatCompletionDelta = choice.delta return chat_completion_delta.content or "" - def _emit_response_completed_event(self, litellm_model_response: ModelResponse) -> Optional[ResponseCompletedEvent]: + def _emit_response_completed_event(self, litellm_model_response: ModelResponse) -> ResponseCompletedEvent | None: if litellm_model_response: # Add cost to usage object if include_cost_in_streaming_usage is True if litellm.include_cost_in_streaming_usage and self.litellm_logging_obj is not None: diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 866698b1f96..1aaa38cea14 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -2,15 +2,17 @@ Handles transforming from Responses API -> LiteLLM completion (Chat Completion API) """ +import json import re from collections.abc import Sequence -from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union, cast +from typing import Any, Literal, cast from openai.types.responses import ResponseFunctionToolCall from openai.types.responses.response_create_params import ResponseInputParam from openai.types.responses.tool_param import FunctionToolParam from typing_extensions import TypedDict +from litellm._logging import verbose_logger from litellm.caching import InMemoryCache from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, @@ -45,6 +47,7 @@ from litellm.types.llms.openai import ( ValidChatCompletionMessageContentTypesLiteral, ) from litellm.types.responses.main import ( + CustomToolCallOutputItem, GenericResponseOutputItem, GenericResponseOutputItemContentAnnotation, OutputCodeInterpreterCall, @@ -62,21 +65,26 @@ from litellm.types.utils import ( Usage, ) +from .custom_tools import ( + convert_custom_tool_to_function_tool, + extract_custom_tool_names, + is_custom_tool_call, + unwrap_custom_tool_arguments, +) + ########### Initialize Classes used for Responses API ########### TOOL_CALLS_CACHE = InMemoryCache() class ChatCompletionSession(TypedDict, total=False): - messages: List[ - Union[ - AllMessageValues, - GenericChatCompletionMessage, - ChatCompletionMessageToolCall, - ChatCompletionResponseMessage, - Message, - ] + messages: list[ + AllMessageValues + | GenericChatCompletionMessage + | ChatCompletionMessageToolCall + | ChatCompletionResponseMessage + | Message ] - litellm_session_id: Optional[str] + litellm_session_id: str | None ########### End of Initialize Classes used for Responses API ########### @@ -109,7 +117,7 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _transform_tool_choice( tool_choice: Any, - ) -> Optional[Union[str, Dict[str, Any]]]: + ) -> str | dict[str, Any] | None: """ Transform tool_choice from various formats to OpenAI Chat Completion format. @@ -159,7 +167,7 @@ class LiteLLMCompletionResponsesConfig: return tool_choice @staticmethod - def _should_drop_derived_web_search_options(model: str, custom_llm_provider: Optional[str]) -> bool: + def _should_drop_derived_web_search_options(model: str, custom_llm_provider: str | None) -> bool: """ A Responses ``web_search`` built-in tool is derived into a ``web_search_options`` param. When the resolved provider/model does not support it (e.g. Bedrock Anthropic, where only @@ -169,7 +177,7 @@ class LiteLLMCompletionResponsesConfig: Support is read from each provider's own ``get_supported_openai_params`` so this bridge stays provider-agnostic; an unmapped provider (``None``) is treated as "keep". """ - supported_params: Optional[List[str]] = get_supported_openai_params( + supported_params: list[str] | None = get_supported_openai_params( model=model, custom_llm_provider=custom_llm_provider ) return supported_params is not None and "web_search_options" not in supported_params @@ -177,11 +185,11 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def transform_responses_api_request_to_chat_completion_request( model: str, - input: Union[str, ResponseInputParam], + input: str | ResponseInputParam, responses_api_request: ResponsesAPIOptionalRequestParams, - custom_llm_provider: Optional[str] = None, - stream: Optional[bool] = None, - extra_headers: Optional[Dict[str, Any]] = None, + custom_llm_provider: str | None = None, + stream: bool | None = None, + extra_headers: dict[str, Any] | None = None, **kwargs, ) -> dict: """ @@ -205,7 +213,7 @@ class LiteLLMCompletionResponsesConfig: response_format = LiteLLMCompletionResponsesConfig._transform_text_format_to_response_format(text_param) # Extract reasoning_effort from reasoning parameter - reasoning_effort: Optional[Union[Reasoning, str]] = None + reasoning_effort: Reasoning | str | None = None reasoning_param = responses_api_request.get("reasoning") if reasoning_param: if isinstance(reasoning_param, dict): @@ -255,7 +263,7 @@ class LiteLLMCompletionResponsesConfig: "include_usage": True, } litellm_completion_request["stream_options"] = stream_options - litellm_logging_obj: Optional[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") + litellm_logging_obj: LiteLLMLoggingObj | None = kwargs.get("litellm_logging_obj") if litellm_logging_obj: litellm_logging_obj.stream_options = stream_options @@ -265,28 +273,24 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def transform_responses_api_input_to_messages( - input: Union[str, ResponseInputParam], - responses_api_request: Union[ResponsesAPIOptionalRequestParams, dict], - ) -> List[ - Union[ - AllMessageValues, - GenericChatCompletionMessage, - ChatCompletionMessageToolCall, - ChatCompletionResponseMessage, - Message, - ] + input: str | ResponseInputParam, + responses_api_request: ResponsesAPIOptionalRequestParams | dict, + ) -> list[ + AllMessageValues + | GenericChatCompletionMessage + | ChatCompletionMessageToolCall + | ChatCompletionResponseMessage + | Message ]: """ Transform a Responses API input into a list of messages """ - messages: List[ - Union[ - AllMessageValues, - GenericChatCompletionMessage, - ChatCompletionMessageToolCall, - ChatCompletionResponseMessage, - Message, - ] + messages: list[ + AllMessageValues + | GenericChatCompletionMessage + | ChatCompletionMessageToolCall + | ChatCompletionResponseMessage + | Message ] = [] if responses_api_request.get("instructions"): messages.append( @@ -373,31 +377,24 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _transform_response_input_param_to_chat_completion_message( - input: Union[str, ResponseInputParam], - ) -> List[ - Union[ - AllMessageValues, - GenericChatCompletionMessage, - ChatCompletionMessageToolCall, - ChatCompletionResponseMessage, - ] + input: str | ResponseInputParam, + ) -> list[ + AllMessageValues | GenericChatCompletionMessage | ChatCompletionMessageToolCall | ChatCompletionResponseMessage ]: """ Transform a ResponseInputParam into a Chat Completion message """ - messages: List[ - Union[ - AllMessageValues, - GenericChatCompletionMessage, - ChatCompletionMessageToolCall, - ChatCompletionResponseMessage, - ] + messages: list[ + AllMessageValues + | GenericChatCompletionMessage + | ChatCompletionMessageToolCall + | ChatCompletionResponseMessage ] = [] if isinstance(input, str): messages.append(ChatCompletionUserMessage(role="user", content=input)) elif isinstance(input, list): - existing_tool_call_ids: Set[str] = set() + existing_tool_call_ids: set[str] = set() for _input in input: chat_completion_messages = ( LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( @@ -449,7 +446,7 @@ class LiteLLMCompletionResponsesConfig: if not chat_completion_messages: continue - deduped_in_place: List[Any] = [] + deduped_in_place: list[Any] = [] for m in chat_completion_messages: role = "" if isinstance(m, dict): @@ -491,36 +488,27 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _deduplicate_tool_call_output_messages( - tool_call_output_messages: List[ - Union[ - AllMessageValues, - GenericChatCompletionMessage, - ChatCompletionMessageToolCall, - ChatCompletionResponseMessage, - ] + tool_call_output_messages: list[ + AllMessageValues + | GenericChatCompletionMessage + | ChatCompletionMessageToolCall + | ChatCompletionResponseMessage ], - existing_tool_call_ids: Set[str], - ) -> List[ - Union[ - AllMessageValues, - GenericChatCompletionMessage, - ChatCompletionMessageToolCall, - ChatCompletionResponseMessage, - ] + existing_tool_call_ids: set[str], + ) -> list[ + AllMessageValues | GenericChatCompletionMessage | ChatCompletionMessageToolCall | ChatCompletionResponseMessage ]: """Return tool call outputs after dropping assistant entries with duplicate call_ids.""" if not tool_call_output_messages: return [] - filtered_messages: List[ - Union[ - AllMessageValues, - GenericChatCompletionMessage, - ChatCompletionMessageToolCall, - ChatCompletionResponseMessage, - ] + filtered_messages: list[ + AllMessageValues + | GenericChatCompletionMessage + | ChatCompletionMessageToolCall + | ChatCompletionResponseMessage ] = [] - seen_tool_call_ids: Set[str] = set(existing_tool_call_ids) + seen_tool_call_ids: set[str] = set(existing_tool_call_ids) for tool_call_message in tool_call_output_messages: if isinstance(tool_call_message, dict): @@ -563,7 +551,7 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _ensure_tool_call_output_has_corresponding_tool_call( - messages: List[Union[AllMessageValues, GenericChatCompletionMessage]], + messages: list[AllMessageValues | GenericChatCompletionMessage], ) -> bool: """ If any tool call output is present, ensure there is a corresponding tool call/tool_use block @@ -574,7 +562,7 @@ class LiteLLMCompletionResponsesConfig: return False @staticmethod - def _find_previous_assistant_idx(messages: List[Any], current_idx: int) -> Optional[int]: + def _find_previous_assistant_idx(messages: list[Any], current_idx: int) -> int | None: """Find the index of the previous assistant message.""" for j in range(current_idx - 1, -1, -1): if messages[j].get("role") == "assistant": @@ -600,7 +588,7 @@ class LiteLLMCompletionResponsesConfig: return "" @staticmethod - def _get_tool_calls_list(assistant_message: Any) -> List[Any]: + def _get_tool_calls_list(assistant_message: Any) -> list[Any]: """Extract tool_calls as a list from assistant message.""" tool_calls_raw = ( assistant_message.get("tool_calls") @@ -616,10 +604,10 @@ class LiteLLMCompletionResponsesConfig: return [] @staticmethod - def _check_tool_call_exists(tool_calls: List[Any], tool_call_id: str) -> bool: + def _check_tool_call_exists(tool_calls: list[Any], tool_call_id: str) -> bool: """Check if a tool_call with the given ID exists in the list.""" for tool_call in tool_calls: - tool_call_id_to_check: Optional[str] = None + tool_call_id_to_check: str | None = None if isinstance(tool_call, dict): tool_call_id_to_check = tool_call.get("id") elif hasattr(tool_call, "id"): @@ -629,7 +617,7 @@ class LiteLLMCompletionResponsesConfig: return False @staticmethod - def _reconstruct_tool_call_from_tools(tool_call_id: str, tools: List[Any]) -> Optional[Dict[str, Any]]: + def _reconstruct_tool_call_from_tools(tool_call_id: str, tools: list[Any]) -> dict[str, Any] | None: """Reconstruct a minimal tool_call definition from tools list.""" for tool in tools: if isinstance(tool, dict): @@ -668,13 +656,13 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _create_tool_call_chunk( - tool_use_definition: Dict[str, Any], tool_call_id: str, index: int + tool_use_definition: dict[str, Any], tool_call_id: str, index: int ) -> ChatCompletionToolCallChunk: """Create a ChatCompletionToolCallChunk from tool_use_definition.""" function_raw = LiteLLMCompletionResponsesConfig._get_mapping_or_attr_value(tool_use_definition, "function") function_name_raw = LiteLLMCompletionResponsesConfig._get_mapping_or_attr_value(function_raw, "name") function_arguments_raw = LiteLLMCompletionResponsesConfig._get_mapping_or_attr_value(function_raw, "arguments") - function: Dict[str, Any] = { + function: dict[str, Any] = { "name": function_name_raw or "", "arguments": function_arguments_raw or "{}", } @@ -693,7 +681,7 @@ class LiteLLMCompletionResponsesConfig: ) @staticmethod - def _normalize_tool_use_definition(tool_use_definition: Any, tool_call_id: str) -> Optional[Dict[str, Any]]: + def _normalize_tool_use_definition(tool_use_definition: Any, tool_call_id: str) -> dict[str, Any] | None: """ Normalize cached tool_call definitions to a dict-like shape consumed by _create_tool_call_chunk. """ @@ -701,7 +689,7 @@ class LiteLLMCompletionResponsesConfig: return None if isinstance(tool_use_definition, dict): - normalized_definition: Dict[str, Any] = dict(tool_use_definition) + normalized_definition: dict[str, Any] = dict(tool_use_definition) else: tool_use_id_raw = LiteLLMCompletionResponsesConfig._get_mapping_or_attr_value(tool_use_definition, "id") tool_use_type_raw = LiteLLMCompletionResponsesConfig._get_mapping_or_attr_value(tool_use_definition, "type") @@ -737,7 +725,7 @@ class LiteLLMCompletionResponsesConfig: def _add_tool_call_to_assistant(assistant_message: Any, tool_call_chunk: ChatCompletionToolCallChunk) -> None: """Add a tool_call to an assistant message.""" if isinstance(assistant_message, dict): - prev_assistant_dict = cast(Dict[str, Any], assistant_message) + prev_assistant_dict = cast(dict[str, Any], assistant_message) if "tool_calls" not in prev_assistant_dict: prev_assistant_dict["tool_calls"] = [] tool_calls_list = prev_assistant_dict["tool_calls"] @@ -752,23 +740,19 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _ensure_tool_results_have_corresponding_tool_calls( messages: Sequence[ - Union[ - AllMessageValues, - GenericChatCompletionMessage, - ChatCompletionResponseMessage, - ChatCompletionMessageToolCall, - Message, - ] + AllMessageValues + | GenericChatCompletionMessage + | ChatCompletionResponseMessage + | ChatCompletionMessageToolCall + | Message ], - tools: Optional[List[Any]] = None, - ) -> List[ - Union[ - AllMessageValues, - GenericChatCompletionMessage, - ChatCompletionResponseMessage, - ChatCompletionMessageToolCall, - Message, - ] + tools: list[Any] | None = None, + ) -> list[ + AllMessageValues + | GenericChatCompletionMessage + | ChatCompletionResponseMessage + | ChatCompletionMessageToolCall + | Message ]: """ Ensure that tool_result messages have corresponding tool_calls in the previous assistant message. @@ -789,14 +773,12 @@ class LiteLLMCompletionResponsesConfig: # Create a deep copy to avoid modifying the original (use list() so we can mutate and return List) import copy - fixed_messages: List[ - Union[ - AllMessageValues, - GenericChatCompletionMessage, - ChatCompletionResponseMessage, - ChatCompletionMessageToolCall, - Message, - ] + fixed_messages: list[ + AllMessageValues + | GenericChatCompletionMessage + | ChatCompletionResponseMessage + | ChatCompletionMessageToolCall + | Message ] = list(copy.deepcopy(messages)) messages_to_remove = [] @@ -828,7 +810,7 @@ class LiteLLMCompletionResponsesConfig: # Type-safe way to set tool_call_id on tool message if isinstance(message, dict): # Cast to dict to allow setting tool_call_id - message_dict = cast(Dict[str, Any], message) + message_dict = cast(dict[str, Any], message) message_dict["tool_call_id"] = tool_call_id elif hasattr(message, "tool_call_id"): setattr(message, "tool_call_id", tool_call_id) @@ -881,13 +863,7 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _transform_responses_api_input_item_to_chat_completion_message( input_item: Any, - ) -> List[ - Union[ - AllMessageValues, - GenericChatCompletionMessage, - ChatCompletionResponseMessage, - ] - ]: + ) -> list[AllMessageValues | GenericChatCompletionMessage | ChatCompletionResponseMessage]: """ Transform a Responses API input item into a Chat Completion message @@ -937,6 +913,7 @@ class LiteLLMCompletionResponsesConfig: """ return input_item.get("type") in [ "function_call_output", + "custom_tool_call_output", "web_search_call", "computer_call_output", "tool_result", # Anthropic/MCP format @@ -945,20 +922,16 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _is_input_item_function_call(input_item: Any) -> bool: """ - Check if the input item is a function call + Check if the input item is a function call or custom tool call. + Both need to be reconstructed as assistant tool_calls for Chat + Completions providers. """ - return input_item.get("type") == "function_call" + return input_item.get("type") in ("function_call", "custom_tool_call") @staticmethod def _transform_responses_api_tool_call_output_to_chat_completion_message( - tool_call_output: Dict[str, Any], - ) -> List[ - Union[ - AllMessageValues, - GenericChatCompletionMessage, - ChatCompletionResponseMessage, - ] - ]: + tool_call_output: dict[str, Any], + ) -> list[AllMessageValues | GenericChatCompletionMessage | ChatCompletionResponseMessage]: """ ChatCompletionToolMessage is used to indicate the output from a tool call """ @@ -992,8 +965,8 @@ class LiteLLMCompletionResponsesConfig: # Some adapters represent tool output as a list of "input_*" parts if isinstance(output, list): - normalized_blocks: List[Dict[str, Any]] = [] - text_acc: List[str] = [] + normalized_blocks: list[dict[str, Any]] = [] + text_acc: list[str] = [] for part in output: if not isinstance(part, dict): continue @@ -1093,14 +1066,8 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _transform_responses_api_function_call_to_chat_completion_message( - function_call: Dict[str, Any], - ) -> List[ - Union[ - AllMessageValues, - GenericChatCompletionMessage, - ChatCompletionResponseMessage, - ] - ]: + function_call: dict[str, Any], + ) -> list[AllMessageValues | GenericChatCompletionMessage | ChatCompletionResponseMessage]: """ Transform a Responses API function_call into a Chat Completion message with tool calls @@ -1117,13 +1084,19 @@ class LiteLLMCompletionResponsesConfig: } ``` """ - # Create a tool call for the function call + # Create a tool call for the function call. Custom tool calls + # store their payload in "input" (raw string) rather than + # "arguments" (JSON string), so normalize to arguments here. + raw_arguments = function_call.get("arguments") + if not raw_arguments and function_call.get("type") == "custom_tool_call": + raw_input = function_call.get("input") or "" + raw_arguments = json.dumps({"content": raw_input}) if raw_input else "" tool_call = ChatCompletionToolCallChunk( id=function_call.get("call_id") or function_call.get("id") or "", type="function", function=ChatCompletionToolCallFunctionChunk( name=function_call.get("name") or "", - arguments=str(function_call.get("arguments") or ""), + arguments=str(raw_arguments or ""), ), index=0, ) @@ -1138,7 +1111,7 @@ class LiteLLMCompletionResponsesConfig: return [chat_completion_response_message] @staticmethod - def _resolve_file_id(item: Dict[str, Any]) -> Optional[str]: + def _resolve_file_id(item: dict[str, Any]) -> str | None: """ Return the effective file_id for a Responses API input_file item. Explicit file_id takes precedence; file_url is used as fallback so @@ -1147,7 +1120,7 @@ class LiteLLMCompletionResponsesConfig: return item.get("file_id") or item.get("file_url") or None @staticmethod - def _transform_input_file_item_to_file_item(item: Dict[str, Any]) -> Dict[str, Any]: + def _transform_input_file_item_to_file_item(item: dict[str, Any]) -> dict[str, Any]: """ Transform a Responses API input_file item to a Chat Completion file item @@ -1157,21 +1130,21 @@ class LiteLLMCompletionResponsesConfig: Returns: Dictionary with transformed file structure for Chat Completion """ - file_dict: Dict[str, Any] = {} + file_dict: dict[str, Any] = {} file_id = LiteLLMCompletionResponsesConfig._resolve_file_id(item) if file_id: file_dict["file_id"] = file_id if item.get("file_data"): file_dict["file_data"] = item["file_data"] - new_item: Dict[str, Any] = {"type": "file", "file": file_dict} + new_item: dict[str, Any] = {"type": "file", "file": file_dict} if "cache_control" in item: new_item["cache_control"] = item["cache_control"] return new_item @staticmethod def _transform_input_image_item_to_image_item( - item: Dict[str, Any], + item: dict[str, Any], ) -> ChatCompletionImageObject: """ Transform a Responses API input_image item to a Chat Completion image item @@ -1185,7 +1158,7 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _transform_responses_api_content_to_chat_completion_content( content: Any, - ) -> Union[str, List[Union[str, Dict[str, Any]]]]: + ) -> str | list[str | dict[str, Any]]: """ Transform a Responses API content into a Chat Completion content @@ -1199,7 +1172,7 @@ class LiteLLMCompletionResponsesConfig: elif isinstance(content, str): return content elif isinstance(content, list): - content_list: List[Union[str, Dict[str, Any]]] = [] + content_list: list[str | dict[str, Any]] = [] for item in content: if isinstance(item, str): content_list.append(item) @@ -1220,7 +1193,7 @@ class LiteLLMCompletionResponsesConfig: text_value = item.get("text") if text_value is None: continue - content_block: Dict[str, Any] = { + content_block: dict[str, Any] = { "type": LiteLLMCompletionResponsesConfig._get_chat_completion_request_content_type( item.get("type") or "text" ), @@ -1268,7 +1241,7 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def transform_instructions_to_system_message( - instructions: Optional[str], + instructions: str | None, ) -> ChatCompletionSystemMessage: """ Transform a Instructions into a system message @@ -1277,18 +1250,18 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def transform_responses_api_tools_to_chat_completion_tools( - tools: Optional[List[Union[FunctionToolParam, OpenAIMcpServerTool]]], - ) -> Tuple[ - List[Union[ChatCompletionToolParam, OpenAIMcpServerTool]], - Optional[OpenAIWebSearchOptions], + tools: list[FunctionToolParam | OpenAIMcpServerTool] | None, + ) -> tuple[ + list[ChatCompletionToolParam | OpenAIMcpServerTool], + OpenAIWebSearchOptions | None, ]: """ Transform a Responses API tools into a Chat Completion tools """ if tools is None: return [], None - chat_completion_tools: List[Union[ChatCompletionToolParam, OpenAIMcpServerTool]] = [] - web_search_options: Optional[OpenAIWebSearchOptions] = None + chat_completion_tools: list[ChatCompletionToolParam | OpenAIMcpServerTool] = [] + web_search_options: OpenAIWebSearchOptions | None = None for tool in tools: if tool.get("type") == "mcp": chat_completion_tools.append(cast(OpenAIMcpServerTool, tool)) @@ -1296,8 +1269,8 @@ class LiteLLMCompletionResponsesConfig: _search_context_size: Literal["low", "medium", "high"] = cast( Literal["low", "medium", "high"], tool.get("search_context_size") ) - _user_location: Optional[OpenAIWebSearchUserLocation] = cast( - Optional[OpenAIWebSearchUserLocation], + _user_location: OpenAIWebSearchUserLocation | None = cast( + OpenAIWebSearchUserLocation | None, tool.get("user_location") or None, ) web_search_options = OpenAIWebSearchOptions( @@ -1310,7 +1283,7 @@ class LiteLLMCompletionResponsesConfig: parameters = dict(typed_tool.get("parameters", {}) or {}) if not parameters or "type" not in parameters: parameters["type"] = "object" - chat_completion_tool: Dict[str, Any] = { + chat_completion_tool: dict[str, Any] = { "type": "function", "function": { "name": typed_tool.get("name") or "", @@ -1328,14 +1301,30 @@ class LiteLLMCompletionResponsesConfig: if tool.get("input_examples"): chat_completion_tool["input_examples"] = tool.get("input_examples") # type: ignore chat_completion_tools.append(cast(ChatCompletionToolParam, chat_completion_tool)) + elif tool.get("type") == "custom": + converted = convert_custom_tool_to_function_tool(tool) + if converted is not None: + chat_completion_tools.append(converted) else: - chat_completion_tools.append(cast(Union[ChatCompletionToolParam, OpenAIMcpServerTool], tool)) + _tool_type = tool.get("type") + if _tool_type in ("computer_use", "image_generation", "namespace", "shell"): + # Drop unsupported Responses-API-only tool types that have no + # Chat Completions equivalent. Passing them through verbatim + # causes providers to reject the request with "'function' is a + # required property". + verbose_logger.warning( + "Dropping Responses API tool of type '%s': it has no Chat Completions " + "equivalent and the target provider would reject the request.", + _tool_type, + ) + continue + chat_completion_tools.append(cast(ChatCompletionToolParam | OpenAIMcpServerTool, tool)) return chat_completion_tools, web_search_options @staticmethod def transform_chat_completion_tool_params_to_responses_api_tools( - chat_completion_tools: Optional[List[Union[ChatCompletionToolParam, OpenAIMcpServerTool]]], - ) -> List[Dict[str, Any]]: + chat_completion_tools: list[ChatCompletionToolParam | OpenAIMcpServerTool] | None, + ) -> list[dict[str, Any]]: """ Transform Chat Completion tool params (e.g. from guardrail output) back to Responses API request tool format. Inverse of @@ -1343,17 +1332,17 @@ class LiteLLMCompletionResponsesConfig: """ if chat_completion_tools is None or not chat_completion_tools: return [] - result: List[Dict[str, Any]] = [] + result: list[dict[str, Any]] = [] for tool in chat_completion_tools: if not isinstance(tool, dict): result.append(tool) # type: ignore continue if tool.get("type") == "function": - fn = cast(Dict[str, Any], tool.get("function") or {}) + fn = cast(dict[str, Any], tool.get("function") or {}) parameters = dict(fn.get("parameters", {}) or {}) if not parameters or "type" not in parameters: parameters["type"] = "object" - responses_tool: Dict[str, Any] = { + responses_tool: dict[str, Any] = { "type": "function", "name": fn.get("name") or "", "description": fn.get("description") or "", @@ -1377,11 +1366,16 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def transform_chat_completion_tools_to_responses_tools( chat_completion_response: ModelResponse, - ) -> List[ResponseFunctionToolCall]: + responses_api_request: ResponsesAPIOptionalRequestParams | None = None, + ) -> list[ResponseFunctionToolCall | CustomToolCallOutputItem]: """ - Transform a Chat Completion tools into a Responses API tools + Transform a Chat Completion tools into a Responses API tools. + + For custom tools (e.g. apply_patch), returns CustomToolCallOutputItem + with ``type="custom_tool_call"``. For regular function tools, returns + ``ResponseFunctionToolCall`` with ``type="function_call"``. """ - all_chat_completion_tools: List[ChatCompletionMessageToolCall] = [] + all_chat_completion_tools: list[ChatCompletionMessageToolCall] = [] for choice in chat_completion_response.choices: if isinstance(choice, Choices): if choice.message.tool_calls: @@ -1392,53 +1386,77 @@ class LiteLLMCompletionResponsesConfig: value=tool_call, ) - responses_tools: List[ResponseFunctionToolCall] = [] + # Extract custom tool names from the original request + custom_tool_names: set[str] = set() + if responses_api_request and "tools" in responses_api_request: + custom_tool_names = extract_custom_tool_names(responses_api_request["tools"]) + + responses_tools: list[ResponseFunctionToolCall | CustomToolCallOutputItem] = [] for tool in all_chat_completion_tools: if tool.type == "function": function_definition = tool.function - provider_specific_fields: Optional[Dict] = None - if hasattr(tool, "provider_specific_fields") and getattr(tool, "provider_specific_fields", None): - provider_specific_fields = getattr(tool, "provider_specific_fields") - if not isinstance(provider_specific_fields, dict): - provider_specific_fields = ( - dict(provider_specific_fields) # type: ignore - if hasattr(provider_specific_fields, "__dict__") - else {} - ) - elif hasattr(function_definition, "provider_specific_fields") and getattr( - function_definition, "provider_specific_fields", None - ): - provider_specific_fields = getattr(function_definition, "provider_specific_fields") - if not isinstance(provider_specific_fields, dict): - provider_specific_fields = ( - dict(provider_specific_fields) # type: ignore - if hasattr(provider_specific_fields, "__dict__") - else {} - ) + tool_name = function_definition.name or "" + tool_id = tool.id or "" + tool_arguments = function_definition.get("arguments") or "" - output_tool_call: ResponseFunctionToolCall = ResponseFunctionToolCall( - name=function_definition.name or "", - arguments=function_definition.get("arguments") or "", - call_id=tool.id or "", - id=tool.id or "", - type="function_call", # critical this is "function_call" to work with tools like openai codex - status=function_definition.get("status") or "completed", - ) + # Check if this is a custom tool + if is_custom_tool_call(tool_name, custom_tool_names): + # Build custom_tool_call output item + input_str = unwrap_custom_tool_arguments(tool_arguments) + custom_item = CustomToolCallOutputItem( + type="custom_tool_call", + call_id=tool_id, + id=tool_id, + name=tool_name, + input=input_str, + status=function_definition.get("status") or "completed", + ) + responses_tools.append(custom_item) + else: + # Build regular function_call output item + provider_specific_fields: dict | None = None + if hasattr(tool, "provider_specific_fields") and getattr(tool, "provider_specific_fields", None): + provider_specific_fields = getattr(tool, "provider_specific_fields") + if not isinstance(provider_specific_fields, dict): + provider_specific_fields = ( + dict(provider_specific_fields) # type: ignore + if hasattr(provider_specific_fields, "__dict__") + else {} + ) + elif hasattr(function_definition, "provider_specific_fields") and getattr( + function_definition, "provider_specific_fields", None + ): + provider_specific_fields = getattr(function_definition, "provider_specific_fields") + if not isinstance(provider_specific_fields, dict): + provider_specific_fields = ( + dict(provider_specific_fields) # type: ignore + if hasattr(provider_specific_fields, "__dict__") + else {} + ) - # Pass through provider_specific_fields as-is if present - if provider_specific_fields: - setattr( - output_tool_call, - "provider_specific_fields", - provider_specific_fields, - ) # type: ignore + output_tool_call: ResponseFunctionToolCall = ResponseFunctionToolCall( + name=tool_name, + arguments=tool_arguments, + call_id=tool_id, + id=tool_id, + type="function_call", + status=function_definition.get("status") or "completed", + ) - responses_tools.append(output_tool_call) + # Pass through provider_specific_fields as-is if present + if provider_specific_fields: + setattr( + output_tool_call, + "provider_specific_fields", + provider_specific_fields, + ) # type: ignore + + responses_tools.append(output_tool_call) return responses_tools @staticmethod def _map_chat_completion_finish_reason_to_responses_status( - finish_reason: Optional[str], + finish_reason: str | None, ) -> ResponsesAPIStatus: """ Map chat completion finish_reason to responses API status. @@ -1465,7 +1483,7 @@ class LiteLLMCompletionResponsesConfig: return "completed" @staticmethod - def _tool_call_id_from_responses_item(item_id: Optional[str], call_id: Optional[str]) -> str: + def _tool_call_id_from_responses_item(item_id: str | None, call_id: str | None) -> str: """Bedrock Mantle returns a non-unique, index-based ``call_id`` (``call_0``, ``call_1``, ... that resets every response) alongside a unique ``id`` (``fc_...``). ``call_id`` is the canonical Responses API correlation key, so @@ -1480,7 +1498,7 @@ class LiteLLMCompletionResponsesConfig: def convert_response_function_tool_call_to_chat_completion_tool_call( tool_call_item: Any, index: int = 0, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: """ Convert ResponseFunctionToolCall to ChatCompletionToolCallChunk format. @@ -1510,7 +1528,7 @@ class LiteLLMCompletionResponsesConfig: ) ) - function_dict: Dict[str, Any] = { + function_dict: dict[str, Any] = { "name": tool_call_item.name, "arguments": tool_call_item.arguments, } @@ -1518,7 +1536,7 @@ class LiteLLMCompletionResponsesConfig: if provider_specific_fields: function_dict["provider_specific_fields"] = provider_specific_fields - tool_call_dict: Dict[str, Any] = { + tool_call_dict: dict[str, Any] = { "id": LiteLLMCompletionResponsesConfig._tool_call_id_from_responses_item( getattr(tool_call_item, "id", None), getattr(tool_call_item, "call_id", None), @@ -1537,7 +1555,7 @@ class LiteLLMCompletionResponsesConfig: def convert_apply_patch_tool_call_to_chat_completion_tool_call( tool_call_item: Any, index: int = 0, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: """ Convert ResponseApplyPatchToolCall to ChatCompletionToolCallChunk format. @@ -1555,7 +1573,7 @@ class LiteLLMCompletionResponsesConfig: import json operation_dict = tool_call_item.operation.model_dump() - tool_call_dict: Dict[str, Any] = { + tool_call_dict: dict[str, Any] = { "id": tool_call_item.call_id, "function": { "name": "apply_patch", @@ -1568,9 +1586,9 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def transform_chat_completion_response_to_responses_api_response( - request_input: Union[str, ResponseInputParam], + request_input: str | ResponseInputParam, responses_api_request: ResponsesAPIOptionalRequestParams, - chat_completion_response: Union[ModelResponse, dict], + chat_completion_response: ModelResponse | dict, ) -> ResponsesAPIResponse: """ Transform a Chat Completion response into a Responses API response @@ -1578,8 +1596,8 @@ class LiteLLMCompletionResponsesConfig: if isinstance(chat_completion_response, dict): chat_completion_response = ModelResponse(**chat_completion_response) # Get finish_reason from the first choice to determine overall status - finish_reason: Optional[str] = None - choices: List[Choices] = getattr(chat_completion_response, "choices", []) + finish_reason: str | None = None + choices: list[Choices] = getattr(chat_completion_response, "choices", []) if choices and len(choices) > 0: finish_reason = choices[0].finish_reason @@ -1595,6 +1613,7 @@ class LiteLLMCompletionResponsesConfig: output=LiteLLMCompletionResponsesConfig._transform_chat_completion_choices_to_responses_output( chat_completion_response=chat_completion_response, choices=getattr(chat_completion_response, "choices", []), + responses_api_request=responses_api_request, ), parallel_tool_calls=getattr(chat_completion_response, "parallel_tool_calls", False), temperature=getattr(chat_completion_response, "temperature", 0), @@ -1626,24 +1645,23 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _transform_chat_completion_choices_to_responses_output( chat_completion_response: ModelResponse, - choices: List[Choices], - ) -> List[ - Union[ - GenericResponseOutputItem, - OutputCodeInterpreterCall, - OutputFunctionToolCall, - OutputImageGenerationCall, - ResponseFunctionToolCall, - ] + choices: list[Choices], + responses_api_request: ResponsesAPIOptionalRequestParams | None = None, + ) -> list[ + GenericResponseOutputItem + | OutputCodeInterpreterCall + | OutputFunctionToolCall + | OutputImageGenerationCall + | ResponseFunctionToolCall + | CustomToolCallOutputItem ]: - responses_output: List[ - Union[ - GenericResponseOutputItem, - OutputCodeInterpreterCall, - OutputFunctionToolCall, - OutputImageGenerationCall, - ResponseFunctionToolCall, - ] + responses_output: list[ + GenericResponseOutputItem + | OutputCodeInterpreterCall + | OutputFunctionToolCall + | OutputImageGenerationCall + | ResponseFunctionToolCall + | CustomToolCallOutputItem ] = [] responses_output.extend( @@ -1654,7 +1672,8 @@ class LiteLLMCompletionResponsesConfig: ) responses_output.extend( LiteLLMCompletionResponsesConfig.transform_chat_completion_tools_to_responses_tools( - chat_completion_response=chat_completion_response + chat_completion_response=chat_completion_response, + responses_api_request=responses_api_request, ) ) @@ -1713,8 +1732,8 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _extract_reasoning_output_items( chat_completion_response: ModelResponse, - choices: List[Choices], - ) -> List[GenericResponseOutputItem]: + choices: list[Choices], + ) -> list[GenericResponseOutputItem]: for choice in choices: if hasattr(choice, "message") and choice.message: message = choice.message @@ -1743,7 +1762,7 @@ class LiteLLMCompletionResponsesConfig: def _extract_image_generation_output_items( chat_completion_response: ModelResponse, choice: Choices, - ) -> List[OutputImageGenerationCall]: + ) -> list[OutputImageGenerationCall]: """ Extract image generation outputs from a choice that contains images. @@ -1762,7 +1781,7 @@ class LiteLLMCompletionResponsesConfig: 'result': 'iVBORw0...' # Pure base64 without data: prefix } """ - image_generation_items: List[OutputImageGenerationCall] = [] + image_generation_items: list[OutputImageGenerationCall] = [] images = getattr(choice.message, "images", []) if not images: @@ -1789,7 +1808,7 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _map_finish_reason_to_image_generation_status( - finish_reason: Optional[str], + finish_reason: str | None, ) -> Literal["in_progress", "completed", "incomplete", "failed"]: """ Map finish_reason to image generation status. @@ -1808,7 +1827,7 @@ class LiteLLMCompletionResponsesConfig: return "completed" @staticmethod - def _extract_base64_from_data_url(data_url: str) -> Optional[str]: + def _extract_base64_from_data_url(data_url: str) -> str | None: """ Extract pure base64 string from a data URL. @@ -1834,9 +1853,9 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _extract_message_output_items( chat_completion_response: ModelResponse, - choices: List[Choices], - ) -> List[Union[GenericResponseOutputItem, OutputImageGenerationCall]]: - message_output_items: List[Union[GenericResponseOutputItem, OutputImageGenerationCall]] = [] + choices: list[Choices], + ) -> list[GenericResponseOutputItem | OutputImageGenerationCall]: + message_output_items: list[GenericResponseOutputItem | OutputImageGenerationCall] = [] for choice in choices: # Check if message has images (image generation) if hasattr(choice.message, "images") and choice.message.images: @@ -1868,20 +1887,8 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _transform_responses_api_outputs_to_chat_completion_messages( responses_api_output: ResponsesAPIResponse, - ) -> List[ - Union[ - AllMessageValues, - GenericChatCompletionMessage, - ChatCompletionMessageToolCall, - ] - ]: - messages: List[ - Union[ - AllMessageValues, - GenericChatCompletionMessage, - ChatCompletionMessageToolCall, - ] - ] = [] + ) -> list[AllMessageValues | GenericChatCompletionMessage | ChatCompletionMessageToolCall]: + messages: list[AllMessageValues | GenericChatCompletionMessage | ChatCompletionMessageToolCall] = [] output_items = responses_api_output.output for _output_item in output_items: output_item: dict = dict(_output_item) @@ -1939,9 +1946,9 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _transform_chat_completion_annotations_to_response_output_annotations( - annotations: Optional[List[ChatCompletionAnnotation]], - ) -> List[GenericResponseOutputItemContentAnnotation]: - response_output_annotations: List[GenericResponseOutputItemContentAnnotation] = [] + annotations: list[ChatCompletionAnnotation] | None, + ) -> list[GenericResponseOutputItemContentAnnotation]: + response_output_annotations: list[GenericResponseOutputItemContentAnnotation] = [] if annotations is None: return response_output_annotations @@ -1965,10 +1972,10 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _transform_chat_completion_usage_to_responses_usage( - chat_completion_response: Union[ModelResponse, Usage], + chat_completion_response: ModelResponse | Usage, ) -> ResponseAPIUsage: if isinstance(chat_completion_response, ModelResponse): - usage: Optional[Usage] = getattr(chat_completion_response, "usage", None) + usage: Usage | None = getattr(chat_completion_response, "usage", None) else: usage = chat_completion_response if usage is None: @@ -1991,7 +1998,7 @@ class LiteLLMCompletionResponsesConfig: # Translate prompt_tokens_details to input_tokens_details if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details is not None: prompt_details = usage.prompt_tokens_details - input_details_dict: Dict[str, int] = {} + input_details_dict: dict[str, int] = {} if hasattr(prompt_details, "cached_tokens") and prompt_details.cached_tokens is not None: input_details_dict["cached_tokens"] = prompt_details.cached_tokens @@ -2010,7 +2017,7 @@ class LiteLLMCompletionResponsesConfig: # Translate completion_tokens_details to output_tokens_details if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details is not None: completion_details = usage.completion_tokens_details - output_details_dict: Dict[str, int] = {} + output_details_dict: dict[str, int] = {} if hasattr(completion_details, "reasoning_tokens") and completion_details.reasoning_tokens is not None: output_details_dict["reasoning_tokens"] = completion_details.reasoning_tokens else: @@ -2029,8 +2036,8 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def _transform_text_format_to_response_format( - text_param: Union[Dict[str, Any], Any], - ) -> Optional[Dict[str, Any]]: + text_param: dict[str, Any] | Any, + ) -> dict[str, Any] | None: """ Transform Responses API text.format parameter to Chat Completion response_format parameter. diff --git a/litellm/router.py b/litellm/router.py index 64b90172c04..12b96430334 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2361,7 +2361,20 @@ class Router: return self async def __anext__(self): - chunk = await self._async_generator.__anext__() + try: + chunk = await self._async_generator.__anext__() + except StopAsyncIteration: + # The inner generator is exhausted. If we never sniffed a + # terminal event off a chunk (the bridge path emits the + # final response.completed via common_done_event_logic, + # which raises StopAsyncIteration after returning it), + # fall back to whatever the source iterator latched so + # the proxy's container-ownership hook still sees a + # completed_response instead of logging a spurious + # "no completed_response" warning. + if self.completed_response is None: + self.completed_response = getattr(source_iterator, "completed_response", None) + raise # Sniff the terminal stream event off each forwarded chunk # so ``self.completed_response`` is populated regardless of # which inner iterator produced it (source_iterator, diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 4c656b32081..3ab5a7b736e 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -90,6 +90,7 @@ from typing_extensions import ( from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject from litellm.types.responses.main import ( + CustomToolCallOutputItem, GenericResponseOutputItem, OutputCodeInterpreterCall, OutputFunctionToolCall, @@ -914,6 +915,7 @@ class OpenAIChatCompletionToolParam(TypedDict): class ChatCompletionToolParam(OpenAIChatCompletionToolParam, total=False): cache_control: ChatCompletionCachedContent + allowed_callers: List[str] class Function(TypedDict, total=False): @@ -1253,6 +1255,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): OutputFunctionToolCall, OutputImageGenerationCall, ResponseFunctionToolCall, + CustomToolCallOutputItem, ] ], ] diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index ebd2ad5b5a8..32e07f9e52f 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -85,6 +85,22 @@ def build_code_interpreter_log_outputs( return [OutputCodeInterpreterCallLog(type="logs", logs=logs)] if logs else None +class CustomToolCallOutputItem(BaseLiteLLMOpenAIResponseObject): + """A custom/freeform tool call output item (e.g. apply_patch). + + Mirrors the ``custom_tool_call`` variant of OpenAI's Responses API output. + Unlike ``OutputFunctionToolCall`` which uses ``arguments`` (JSON string), + this uses ``input`` (raw string) for the tool payload. + """ + + type: Literal["custom_tool_call"] + call_id: str + id: Optional[str] = None + name: str + input: str + status: Optional[Literal["in_progress", "completed", "incomplete"]] = None + + class GenericResponseOutputItem(BaseLiteLLMOpenAIResponseObject): """ Generic response API output item diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index c185b694e68..c61fdec370c 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -3304,6 +3304,75 @@ class TestStreamingClientDisconnectLogging: assert recorded is False assert "client_disconnected" not in request_data["metadata"] + @pytest.mark.asyncio + async def test_record_streaming_client_disconnect_handles_none_metadata(self): + from litellm.proxy.common_request_processing import ( + _record_streaming_client_disconnect_if_needed, + ) + + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = { + "litellm_params": {"metadata": None}, + "metadata": None, + } + mock_request = MagicMock(spec=Request) + mock_request.is_disconnected = AsyncMock(return_value=True) + request_data = { + "litellm_call_id": "test-call-id", + "litellm_logging_obj": mock_logging_obj, + "metadata": {}, + "litellm_params": {"metadata": {}}, + } + + recorded = await _record_streaming_client_disconnect_if_needed( + mock_request, request_data + ) + + assert recorded is True + assert request_data["metadata"]["client_disconnected"] is True + assert ( + mock_logging_obj.model_call_details["litellm_params"]["metadata"][ + "client_disconnected" + ] + is True + ) + assert ( + mock_logging_obj.model_call_details["metadata"]["client_disconnected"] + is True + ) + + @pytest.mark.asyncio + async def test_record_streaming_client_disconnect_handles_none_request_data_metadata(self): + from litellm.proxy.common_request_processing import ( + _record_streaming_client_disconnect_if_needed, + ) + + mock_request = MagicMock(spec=Request) + mock_request.is_disconnected = AsyncMock(return_value=True) + request_data = { + "litellm_call_id": "test-call-id", + "metadata": None, + "litellm_params": {"metadata": None}, + } + + recorded = await _record_streaming_client_disconnect_if_needed( + mock_request, request_data + ) + + assert recorded is True + assert request_data["metadata"]["client_disconnected"] is True + assert ( + request_data["litellm_params"]["metadata"]["client_disconnected"] is True + ) + + @pytest.mark.asyncio + async def test_apply_client_disconnect_metadata_none_returns_early(self): + from litellm.proxy.common_request_processing import ( + _apply_client_disconnect_metadata, + ) + + _apply_client_disconnect_metadata(None) + @pytest.mark.asyncio async def test_finalize_streaming_generator_cleanup_fires_deferred_logging( self, monkeypatch diff --git a/tests/test_litellm/proxy/test_tools_allowlist_enforcement.py b/tests/test_litellm/proxy/test_tools_allowlist_enforcement.py index 8196cc97f50..31f5fbf606b 100644 --- a/tests/test_litellm/proxy/test_tools_allowlist_enforcement.py +++ b/tests/test_litellm/proxy/test_tools_allowlist_enforcement.py @@ -70,6 +70,22 @@ class TestExtractRequestToolNames: } assert extract_request_tool_names("/v1/responses", data) == ["dmcp"] + def test_openai_responses_custom_tools(self): + """Custom tools become callable function tools on the Chat Completions + bridge, so their names must be extracted for allowlist enforcement; + otherwise a restricted key could invoke a disallowed tool by declaring + it with type "custom" (VERIA finding on PR #32258).""" + data = { + "tools": [ + {"type": "custom", "name": "apply_patch", "description": "x"}, + {"type": "function", "name": "get_current_weather"}, + ] + } + assert extract_request_tool_names("/v1/responses", data) == [ + "apply_patch", + "get_current_weather", + ] + def test_anthropic_tools(self): data = {"tools": [{"name": "get_weather"}, {"name": "run_sql"}]} assert extract_request_tool_names("/v1/messages", data) == [ @@ -143,6 +159,20 @@ class TestCheckToolsAllowlist: assert exc_info.value.type == ProxyErrorTypes.tool_access_denied assert "get_weather" in str(exc_info.value.message) + @pytest.mark.asyncio + async def test_disallowed_custom_tool_raises_on_responses_route(self): + token = _token(metadata={"allowed_tools": ["other_tool"]}) + body = {"tools": [{"type": "custom", "name": "restricted_tool"}]} + with pytest.raises(ProxyException) as exc_info: + await check_tools_allowlist( + request_body=body, + valid_token=token, + team_object=None, + route="/v1/responses", + ) + assert exc_info.value.type == ProxyErrorTypes.tool_access_denied + assert "restricted_tool" in str(exc_info.value.message) + @pytest.mark.asyncio async def test_team_allowlist_used_when_key_empty(self): token = _token( diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 426c73645c1..9cf60ea5f91 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1,6 +1,8 @@ import os import sys +import pytest + sys.path.insert( 0, os.path.abspath("../../..") ) # Adds the parent directory to the system path @@ -1066,7 +1068,13 @@ class TestToolTransformation: assert web_search_options is None def test_transform_computer_use_tools(self): - """Test that computer_use tools are passed through as-is""" + """Test that computer_use tools are dropped (no Chat Completions equivalent). + + This deliberately reverses the previous pass-through regression guard: + forwarding computer_use verbatim made Chat Completions providers reject + the whole request with "'function' is a required property", so the + bridge now drops such tools (with a warning log) instead. + """ computer_use_tool = { "type": "computer_use", "display_width_px": 1024, @@ -1083,11 +1091,129 @@ class TestToolTransformation: tools=tools ) + # Assert - computer_use has no Chat Completions equivalent, so it is dropped + assert len(result_tools) == 0 + assert web_search_options is None + + def test_transform_custom_tools_to_function_tools(self): + """Test that custom (freeform/grammar) tools are converted to function tools""" + custom_tool = { + "type": "custom", + "name": "apply_patch", + "description": "Apply a patch to files", + "format": { + "type": "grammar", + "syntax": "lark", + "definition": "start: begin_patch hunk+ end_patch", + }, + } + + tools = [custom_tool] + + # Execute + ( + result_tools, + web_search_options, + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) + + # Assert - custom tool is converted to a function tool + assert len(result_tools) == 1 + assert result_tools[0]["type"] == "function" + assert result_tools[0]["function"]["name"] == "apply_patch" + assert "content" in result_tools[0]["function"]["parameters"]["properties"] + assert result_tools[0]["function"]["parameters"]["required"] == ["content"] + assert "begin_patch" in result_tools[0]["function"]["description"] + assert web_search_options is None + + def test_transform_custom_tools_without_format(self): + """Test that custom tools without format info are still converted""" + custom_tool = { + "type": "custom", + "name": "exec", + "description": "Execute code", + } + + tools = [custom_tool] + + # Execute + ( + result_tools, + web_search_options, + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) + # Assert assert len(result_tools) == 1 - assert result_tools[0] == computer_use_tool - assert result_tools[0]["type"] == "computer_use" - assert web_search_options is None + assert result_tools[0]["type"] == "function" + assert result_tools[0]["function"]["name"] == "exec" + assert result_tools[0]["function"]["description"] == "Execute code" + + def test_transform_custom_tools_preserves_allowed_callers(self): + """allowed_callers on a custom tool gates direct model invocation in the + Anthropic adapter, so it must survive the custom->function conversion.""" + custom_tool = { + "type": "custom", + "name": "apply_patch", + "description": "Apply a patch to files", + "allowed_callers": ["exec"], + } + + tools = [custom_tool] + + # Execute + ( + result_tools, + _, + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) + + # Assert + assert len(result_tools) == 1 + assert result_tools[0]["type"] == "function" + assert result_tools[0]["allowed_callers"] == ["exec"] + + def test_transform_custom_tools_without_allowed_callers(self): + """A custom tool without allowed_callers must not synthesize the field.""" + custom_tool = { + "type": "custom", + "name": "apply_patch", + "description": "Apply a patch to files", + } + + tools = [custom_tool] + + # Execute + ( + result_tools, + _, + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) + + # Assert + assert len(result_tools) == 1 + assert "allowed_callers" not in result_tools[0] + + def test_transform_custom_tools_rejects_invalid_allowed_callers(self): + """Invalid allowed_callers must raise rather than silently dropping the + provider-side allowlist.""" + custom_tool = { + "type": "custom", + "name": "apply_patch", + "description": "Apply a patch to files", + "allowed_callers": "exec", + } + + tools = [custom_tool] + + with pytest.raises(ValueError, match="allowed_callers must be a list of strings"): + LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) def test_transform_web_search_tools_to_web_search_options(self): """Test that web_search tools are converted to web_search_options""" @@ -2185,6 +2311,155 @@ class TestStreamingIDConsistency: assert tool_calls is not None and len(tool_calls) == 1 +class TestCompletedResponseLatchedOnStreamEnd: + """Regression: LiteLLMCompletionStreamingIterator (the Chat Completions + bridge path) never set ``self.completed_response`` because it overrides + __anext__ and bypasses the base class's _process_chunk where that + attribute is normally latched. FallbackResponsesStreamWrapper reads + ``completed_response`` via getattr to record container ownership; when + it stays None the proxy logs a "Container ownership recording skipped" + warning and follow-up /v1/containers//files calls 403 for non-admin + keys. Codex CLI's apply_patch tool also surfaces as "aborted" because + the terminal response.completed event never propagates correctly.""" + + def _make_iterator_with_stop(self, model_response): + """Build a LiteLLMCompletionStreamingIterator whose underlying + CustomStreamWrapper raises StopAsyncIteration immediately (simulating + a stream that already delivered all content chunks).""" + from unittest.mock import Mock + + import litellm + from litellm.responses.litellm_completion_transformation.streaming_iterator import ( + LiteLLMCompletionStreamingIterator, + ) + + mock_wrapper = Mock(spec=litellm.CustomStreamWrapper) + mock_wrapper.logging_obj = Mock() + mock_wrapper.logging_obj._response_cost_calculator = Mock(return_value=0.0) + mock_wrapper.__aiter__ = Mock(return_value=mock_wrapper) + mock_wrapper.__anext__ = Mock(side_effect=StopAsyncIteration) + + iterator = LiteLLMCompletionStreamingIterator( + model="deepseek/deepseek-chat", + litellm_custom_stream_wrapper=mock_wrapper, + request_input="test", + responses_api_request={}, + ) + iterator.litellm_model_response = model_response + return iterator + + def test_completed_response_set_after_common_done_event_logic(self): + """common_done_event_logic builds a ResponseCompletedEvent and must + latch it onto self.completed_response so downstream wrappers can + read it. Before the fix the event was returned but + completed_response stayed None.""" + from litellm.types.utils import Choices, Message, ModelResponse + + complete_response = ModelResponse( + id="resp_test", + created=1234567890, + model="deepseek-chat", + object="chat.completion", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message(content="hello", role="assistant"), + ) + ], + ) + iterator = self._make_iterator_with_stop(complete_response) + + import asyncio + + async def drain(): + results = [] + try: + async for chunk in iterator: + results.append(chunk) + except StopAsyncIteration: + pass + return results + + results = asyncio.run(drain()) + assert len(results) > 0 + assert iterator.completed_response is not None, ( + "LiteLLMCompletionStreamingIterator.completed_response is still None " + "after common_done_event_logic ran — downstream wrappers and the " + "proxy container-ownership hook will see no terminal event" + ) + assert iterator.completed_response.type == "response.completed" + + +class TestFallbackWrapperStopAsyncIterationFallback: + """Regression: FallbackResponsesStreamWrapper.__anext__ only sniffed + terminal events off forwarded chunks. When the inner generator raises + StopAsyncIteration without a sniffable chunk (the bridge path ends this + way), the wrapper re-raised without checking source_iterator for a + latched completed_response, leaving its own completed_response None.""" + + def test_falls_back_to_source_completed_response_on_stop(self): + """When the inner async generator raises StopAsyncIteration and the + wrapper never sniffed a terminal chunk, it must copy + source_iterator.completed_response so the proxy ownership hook + still sees the terminal event.""" + import asyncio + from types import SimpleNamespace + + from litellm.router import Router + + source = SimpleNamespace( + response=None, + model="deepseek/deepseek-chat", + logging_obj=None, + responses_api_provider_config=None, + start_time=None, + litellm_metadata=None, + custom_llm_provider="deepseek", + request_data={}, + call_type="aresponses", + _hidden_params={}, + completed_response=SimpleNamespace( + type="response.completed", + response=SimpleNamespace(id="resp_src", output=[], container=None), + ), + ) + + async def empty_gen(): + return + yield # pragma: no cover + + async def _drive(): + router = Router( + model_list=[ + { + "model_name": "deepseek/deepseek-chat", + "litellm_params": {"model": "deepseek/deepseek-chat", "api_key": "sk-test"}, + } + ] + ) + wrapper = await router._aresponses_streaming_iterator( + response=source, # type: ignore[arg-type] + initial_kwargs={}, + ) + wrapper._async_generator = empty_gen() + out = [] + try: + async for chunk in wrapper: + out.append(chunk) + except StopAsyncIteration: + pass + return wrapper, out + + wrapper, _ = asyncio.run(_drive()) + assert wrapper.completed_response is not None, ( + "FallbackResponsesStreamWrapper.completed_response is None after " + "StopAsyncIteration even though source_iterator had one — the " + "proxy ownership hook will log a spurious warning" + ) + assert wrapper.completed_response.type == "response.completed" + + class TestEnsureOutputItemContentPartAdded: """Test that _ensure_output_item_for_chunk emits content_part.added after output_item.added for message items.""" diff --git a/tests/test_litellm/responses/test_custom_tool_call.py b/tests/test_litellm/responses/test_custom_tool_call.py new file mode 100644 index 00000000000..c605ef24934 --- /dev/null +++ b/tests/test_litellm/responses/test_custom_tool_call.py @@ -0,0 +1,343 @@ +""" +Test custom_tool_call adaptation for apply_patch and other custom tools. + +This test verifies that when Codex sends custom tools (type="custom"), +LiteLLM bridge correctly: +1. Converts them to function tools for Chat Completions providers +2. Converts function_call responses back to custom_tool_call output items +3. Unwraps the JSON-wrapping arguments to extract the actual input content +""" + +import json +import pytest +from typing import Dict, Any, List + +from openai.types.responses import ResponseFunctionToolCall + +from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, +) +from litellm.responses.litellm_completion_transformation.custom_tools import ( + extract_custom_tool_names, + is_custom_tool_call, + unwrap_custom_tool_arguments, + build_tool_call_item_kwargs, + convert_custom_tool_to_function_tool, + _MAX_ARGUMENTS_LEN, +) + +from litellm.types.responses.main import CustomToolCallOutputItem + + +class TestCustomToolUtilities: + """Test the custom_tools utility functions.""" + + def test_extract_custom_tool_names(self): + """Test extraction of custom tool names from tools list.""" + tools = [ + {"type": "function", "name": "regular_tool"}, + {"type": "custom", "name": "apply_patch"}, + {"type": "function", "name": "another_tool"}, + {"type": "custom", "name": "custom_format"}, + ] + + names = extract_custom_tool_names(tools) + assert names == {"apply_patch", "custom_format"} + + def test_extract_custom_tool_names_empty(self): + """Test extraction with no custom tools.""" + tools = [ + {"type": "function", "name": "tool1"}, + {"type": "function", "name": "tool2"}, + ] + + names = extract_custom_tool_names(tools) + assert names == set() + + def test_extract_custom_tool_names_none(self): + """Test extraction with None input.""" + names = extract_custom_tool_names(None) + assert names == set() + + def test_is_custom_tool_call_true(self): + """Test identification of custom tool call.""" + custom_names = {"apply_patch", "custom_format"} + assert is_custom_tool_call("apply_patch", custom_names) is True + assert is_custom_tool_call("custom_format", custom_names) is True + + def test_is_custom_tool_call_false(self): + """Test identification of non-custom tool call.""" + custom_names = {"apply_patch"} + assert is_custom_tool_call("regular_tool", custom_names) is False + assert is_custom_tool_call("unknown_tool", custom_names) is False + + def test_unwrap_custom_tool_arguments(self): + """Test unwrapping of JSON-wrapped arguments.""" + # Test with valid JSON + wrapped = json.dumps({"content": "*** Begin Patch\n*** Add File: test.py\n+hello\n*** End Patch"}) + unwrapped = unwrap_custom_tool_arguments(wrapped) + assert unwrapped == "*** Begin Patch\n*** Add File: test.py\n+hello\n*** End Patch" + + def test_unwrap_custom_tool_arguments_invalid_json(self): + """Test unwrapping with invalid JSON returns original.""" + raw = "*** Begin Patch\n*** Add File: test.py\n+hello\n*** End Patch" + unwrapped = unwrap_custom_tool_arguments(raw) + assert unwrapped == raw + + def test_unwrap_custom_tool_arguments_no_content_key(self): + """Test unwrapping with JSON but no content key.""" + wrapped = json.dumps({"other_key": "value"}) + unwrapped = unwrap_custom_tool_arguments(wrapped) + assert unwrapped == wrapped + + def test_build_tool_call_item_kwargs_custom_completed(self): + """A completed custom tool call unwraps the content into `input`.""" + wrapped = json.dumps({"content": "patch body"}) + kwargs = build_tool_call_item_kwargs( + call_id="c1", + name="apply_patch", + arguments_or_input=wrapped, + status="completed", + custom_tool_names={"apply_patch"}, + ) + assert kwargs["type"] == "custom_tool_call" + assert kwargs["input"] == "patch body" + assert "arguments" not in kwargs + + def test_build_tool_call_item_kwargs_custom_in_progress(self): + """An in-progress custom tool call seeds an empty input string.""" + kwargs = build_tool_call_item_kwargs( + call_id="c2", + name="apply_patch", + arguments_or_input="ignored-until-completed", + status="in_progress", + custom_tool_names={"apply_patch"}, + ) + assert kwargs["input"] == "" + + def test_build_tool_call_item_kwargs_regular_function(self): + """A regular function call keeps raw arguments and uses function_call type.""" + raw = json.dumps({"k": "v"}) + kwargs = build_tool_call_item_kwargs( + call_id="c3", + name="get_weather", + arguments_or_input=raw, + status="completed", + custom_tool_names=set(), + ) + assert kwargs["type"] == "function_call" + assert kwargs["arguments"] == raw + assert "input" not in kwargs + + def test_unwrap_custom_tool_arguments_oversized_returns_raw(self): + """Arguments larger than the safety cap are returned unchanged to avoid + OOM on JSON parsing a pathologically large string.""" + oversized = "x" * (_MAX_ARGUMENTS_LEN + 1) + assert unwrap_custom_tool_arguments(oversized) == oversized + + def test_unwrap_custom_tool_arguments_empty(self): + """Empty arguments unwrap to an empty string, not the raw input.""" + assert unwrap_custom_tool_arguments("") == "" + + def test_convert_custom_tool_to_function_tool_with_format(self): + """The grammar definition is embedded in the description so the model can + produce correctly-formatted output.""" + tool = { + "type": "custom", + "name": "apply_patch", + "description": "Apply a patch", + "format": { + "type": "grammar", + "syntax": "lark", + "definition": "start: begin_patch", + }, + } + result = convert_custom_tool_to_function_tool(tool) + assert result is not None + assert result["type"] == "function" + assert "begin_patch" in result["function"]["description"] + assert result["function"]["parameters"]["required"] == ["content"] + + def test_convert_custom_tool_to_function_tool_non_custom_returns_none(self): + """Non-custom tools are not convertible; the caller keeps them as-is.""" + assert convert_custom_tool_to_function_tool({"type": "function"}) is None + + +class TestTransformationCustomTools: + """Test custom tool handling in transformation logic.""" + + def test_transform_apply_patch_function_call_to_custom_tool_call(self): + """Test that apply_patch function_call is converted to custom_tool_call.""" + # Simulate a Chat Completion response with apply_patch function call + from litellm.types.utils import ModelResponse, Choices, Message, ChatCompletionMessageToolCall, Function + + tool_call = ChatCompletionMessageToolCall( + id="call_abc123", + type="function", + function=Function( + name="apply_patch", + arguments=json.dumps({"content": "*** Begin Patch\n*** Add File: test.py\n+hello\n*** End Patch"}), + ), + ) + + message = Message(role="assistant", content=None, tool_calls=[tool_call]) + + choices = [Choices(index=0, message=message, finish_reason="tool_calls")] + + response = ModelResponse( + id="test_response", choices=choices, created=1234567890, model="gpt-4", object="chat.completion" + ) + + # Transform with custom tool names + responses_api_request = { + "tools": [{"type": "custom", "name": "apply_patch"}, {"type": "function", "name": "regular_tool"}] + } + + result = LiteLLMCompletionResponsesConfig.transform_chat_completion_tools_to_responses_tools( + response, responses_api_request=responses_api_request + ) + + # Should return a CustomToolCallOutputItem object; ResponsesAPIResponse + # accepts it directly via its output item union. + assert len(result) == 1 + item = result[0] + assert isinstance(item, CustomToolCallOutputItem) + assert item.type == "custom_tool_call" + assert item.call_id == "call_abc123" + assert item.name == "apply_patch" + assert item.input == "*** Begin Patch\n*** Add File: test.py\n+hello\n*** End Patch" + assert item.status == "completed" + + def test_custom_tool_call_input_item_recovers_payload_from_input(self): + """A custom_tool_call input item stores its payload in `input`; the + assistant tool call must carry it as a JSON content envelope whether + `arguments` is missing or an empty string.""" + for arguments in (None, ""): + item = { + "type": "custom_tool_call", + "call_id": "call_1", + "name": "apply_patch", + "input": "*** Begin Patch\n+hello\n*** End Patch", + } + if arguments is not None: + item["arguments"] = arguments + messages = ( + LiteLLMCompletionResponsesConfig._transform_responses_api_function_call_to_chat_completion_message( + function_call=item + ) + ) + tool_call = messages[0]["tool_calls"][0] + assert tool_call["function"]["arguments"] == json.dumps( + {"content": "*** Begin Patch\n+hello\n*** End Patch"} + ) + + def test_function_call_input_item_with_empty_arguments_keeps_them_empty(self): + """A plain function_call input item with empty or missing `arguments` + must produce an empty arguments string, never a `{"content": ...}` + envelope (that recovery is reserved for custom_tool_call items) and + never the literal string "None".""" + for item in ( + { + "type": "function_call", + "call_id": "call_2", + "name": "get_weather", + "arguments": "", + "input": "stray value", + }, + { + "type": "function_call", + "call_id": "call_3", + "name": "get_weather", + }, + ): + messages = ( + LiteLLMCompletionResponsesConfig._transform_responses_api_function_call_to_chat_completion_message( + function_call=item + ) + ) + tool_call = messages[0]["tool_calls"][0] + assert tool_call["function"]["arguments"] == "" + + def test_transform_regular_function_call_unchanged(self): + """Test that regular function calls remain as ResponseFunctionToolCall.""" + from litellm.types.utils import ModelResponse, Choices, Message, ChatCompletionMessageToolCall, Function + + tool_call = ChatCompletionMessageToolCall( + id="call_xyz789", + type="function", + function=Function(name="regular_tool", arguments=json.dumps({"param": "value"})), + ) + + message = Message(role="assistant", content=None, tool_calls=[tool_call]) + + choices = [Choices(index=0, message=message, finish_reason="tool_calls")] + + response = ModelResponse( + id="test_response", choices=choices, created=1234567890, model="gpt-4", object="chat.completion" + ) + + # Transform with custom tool names (regular_tool is NOT custom) + responses_api_request = { + "tools": [{"type": "custom", "name": "apply_patch"}, {"type": "function", "name": "regular_tool"}] + } + + result = LiteLLMCompletionResponsesConfig.transform_chat_completion_tools_to_responses_tools( + response, responses_api_request=responses_api_request + ) + + # Should return ResponseFunctionToolCall + assert len(result) == 1 + item = result[0] + assert isinstance(item, ResponseFunctionToolCall) + assert item.type == "function_call" + assert item.name == "regular_tool" + assert item.arguments == json.dumps({"param": "value"}) + + def test_transform_mixed_tool_calls(self): + """Test transformation with both custom and regular tool calls.""" + from litellm.types.utils import ModelResponse, Choices, Message, ChatCompletionMessageToolCall, Function + + custom_call = ChatCompletionMessageToolCall( + id="call_001", + type="function", + function=Function(name="apply_patch", arguments=json.dumps({"content": "patch content"})), + ) + + regular_call = ChatCompletionMessageToolCall( + id="call_002", type="function", function=Function(name="get_weather", arguments=json.dumps({"city": "SF"})) + ) + + message = Message(role="assistant", content=None, tool_calls=[custom_call, regular_call]) + + choices = [Choices(index=0, message=message, finish_reason="tool_calls")] + + response = ModelResponse( + id="test_response", choices=choices, created=1234567890, model="gpt-4", object="chat.completion" + ) + + responses_api_request = { + "tools": [{"type": "custom", "name": "apply_patch"}, {"type": "function", "name": "get_weather"}] + } + + result = LiteLLMCompletionResponsesConfig.transform_chat_completion_tools_to_responses_tools( + response, responses_api_request=responses_api_request + ) + + assert len(result) == 2 + + # First should be custom_tool_call object + first = result[0] + assert isinstance(first, CustomToolCallOutputItem) + assert first.type == "custom_tool_call" + assert first.name == "apply_patch" + assert first.input == "patch content" + + # Second should be function_call + second = result[1] + assert isinstance(second, ResponseFunctionToolCall) + assert second.type == "function_call" + assert second.name == "get_weather" + + +if __name__ == "__main__": + pytest.main([__file__, "-v"])