diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 7e5a4f22a7f..6fe0fcd4fdf 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -25,8 +25,18 @@ from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager, client from ..adapters.handler import LiteLLMMessagesToCompletionTransformationHandler +from ..responses_adapters.handler import LiteLLMMessagesToResponsesAPIHandler from .utils import AnthropicMessagesRequestUtils, mock_response +# Providers that are routed directly to the OpenAI Responses API instead of +# going through chat/completions. +_RESPONSES_API_PROVIDERS = frozenset({"openai", "azure", "azure_text"}) + + +def _should_route_to_responses_api(custom_llm_provider: Optional[str]) -> bool: + """Return True when the provider should use the Responses API path.""" + return custom_llm_provider in _RESPONSES_API_PROVIDERS + ####### ENVIRONMENT VARIABLES ################### # Initialize any necessary instances or variables here base_llm_http_handler = BaseLLMHTTPHandler() @@ -282,29 +292,34 @@ def anthropic_messages_handler( ) ) if anthropic_messages_provider_config is None: - # Handle non-Anthropic models using the adapter - return ( - LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( - max_tokens=max_tokens, - messages=messages, - model=model, - metadata=metadata, - stop_sequences=stop_sequences, - stream=stream, - system=system, - temperature=temperature, - thinking=thinking, - tool_choice=tool_choice, - tools=tools, - top_k=top_k, - top_p=top_p, - _is_async=is_async, - api_key=api_key, - api_base=api_base, - client=client, - custom_llm_provider=custom_llm_provider, - **kwargs, + # Route to Responses API for OpenAI / Azure, chat/completions for everything else. + _shared_kwargs = dict( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + _is_async=is_async, + api_key=api_key, + api_base=api_base, + client=client, + custom_llm_provider=custom_llm_provider, + **kwargs, + ) + if _should_route_to_responses_api(custom_llm_provider): + return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + **_shared_kwargs ) + return LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( + **_shared_kwargs ) if custom_llm_provider is None: diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py new file mode 100644 index 00000000000..6ad3c7b0164 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py @@ -0,0 +1,3 @@ +from .transformation import LiteLLMAnthropicToResponsesAPIAdapter + +__all__ = ["LiteLLMAnthropicToResponsesAPIAdapter"] diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py new file mode 100644 index 00000000000..18dbabb1e14 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py @@ -0,0 +1,213 @@ +""" +Handler for the Anthropic v1/messages -> OpenAI Responses API path. + +Used when the target model is an OpenAI or Azure model. +""" + +from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union + +import litellm +from litellm.types.llms.anthropic import AnthropicMessagesRequest +from litellm.types.llms.anthropic_messages.anthropic_response import ( + AnthropicMessagesResponse, +) +from litellm.types.llms.openai import ResponsesAPIResponse + +from .streaming_iterator import AnthropicResponsesStreamWrapper +from .transformation import LiteLLMAnthropicToResponsesAPIAdapter + +_ADAPTER = LiteLLMAnthropicToResponsesAPIAdapter() + + +def _build_responses_kwargs( + *, + max_tokens: int, + messages: List[Dict], + model: str, + metadata: Optional[Dict] = None, + stop_sequences: Optional[List[str]] = None, + stream: Optional[bool] = False, + system: Optional[str] = None, + temperature: Optional[float] = None, + thinking: Optional[Dict] = None, + tool_choice: Optional[Dict] = None, + tools: Optional[List[Dict]] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + output_format: Optional[Dict] = None, + extra_kwargs: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """ + Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses(). + """ + # Build a typed AnthropicMessagesRequest for the adapter + request_data: Dict[str, Any] = {"model": model, "messages": messages, "max_tokens": max_tokens} + if metadata: + request_data["metadata"] = metadata + if system: + request_data["system"] = system + if temperature is not None: + request_data["temperature"] = temperature + if thinking: + request_data["thinking"] = thinking + if tool_choice: + request_data["tool_choice"] = tool_choice + if tools: + request_data["tools"] = tools + if top_p is not None: + request_data["top_p"] = top_p + if output_format: + request_data["output_format"] = output_format + + anthropic_request = AnthropicMessagesRequest(**request_data) + responses_kwargs = _ADAPTER.translate_request(anthropic_request) + + if stream: + responses_kwargs["stream"] = True + + # Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.) + excluded = {"anthropic_messages"} + for key, value in (extra_kwargs or {}).items(): + if key == "litellm_logging_obj" and value is not None: + from litellm.litellm_core_utils.litellm_logging import ( + Logging as LiteLLMLoggingObject, + ) + from litellm.types.utils import CallTypes + + if isinstance(value, LiteLLMLoggingObject): + # Reclassify as acompletion so the success handler doesn't try to + # validate the Responses API event as an AnthropicResponse. + # (Mirrors the pattern used in LiteLLMMessagesToCompletionTransformationHandler.) + setattr(value, "call_type", CallTypes.acompletion.value) + responses_kwargs[key] = value + elif key not in excluded and key not in responses_kwargs and value is not None: + responses_kwargs[key] = value + + return responses_kwargs + + +class LiteLLMMessagesToResponsesAPIHandler: + """ + Handles Anthropic /v1/messages requests for OpenAI / Azure models by + calling litellm.responses() / litellm.aresponses() directly and translating + the response back to Anthropic format. + """ + + @staticmethod + async def async_anthropic_messages_handler( + max_tokens: int, + messages: List[Dict], + model: str, + metadata: Optional[Dict] = None, + stop_sequences: Optional[List[str]] = None, + stream: Optional[bool] = False, + system: Optional[str] = None, + temperature: Optional[float] = None, + thinking: Optional[Dict] = None, + tool_choice: Optional[Dict] = None, + tools: Optional[List[Dict]] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + output_format: Optional[Dict] = None, + **kwargs, + ) -> Union[AnthropicMessagesResponse, AsyncIterator]: + responses_kwargs = _build_responses_kwargs( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + extra_kwargs=kwargs, + ) + + result = await litellm.aresponses(**responses_kwargs) + + if stream: + wrapper = AnthropicResponsesStreamWrapper(responses_stream=result, model=model) + return wrapper.async_anthropic_sse_wrapper() + + if not isinstance(result, ResponsesAPIResponse): + raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") + + return _ADAPTER.translate_response(result) + + @staticmethod + def anthropic_messages_handler( + max_tokens: int, + messages: List[Dict], + model: str, + metadata: Optional[Dict] = None, + stop_sequences: Optional[List[str]] = None, + stream: Optional[bool] = False, + system: Optional[str] = None, + temperature: Optional[float] = None, + thinking: Optional[Dict] = None, + tool_choice: Optional[Dict] = None, + tools: Optional[List[Dict]] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + output_format: Optional[Dict] = None, + _is_async: bool = False, + **kwargs, + ) -> Union[ + AnthropicMessagesResponse, + AsyncIterator[Any], + Coroutine[Any, Any, Union[AnthropicMessagesResponse, AsyncIterator[Any]]], + ]: + if _is_async: + return LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + **kwargs, + ) + + # Sync path + responses_kwargs = _build_responses_kwargs( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + extra_kwargs=kwargs, + ) + + result = litellm.responses(**responses_kwargs) + + if stream: + wrapper = AnthropicResponsesStreamWrapper(responses_stream=result, model=model) + return wrapper.async_anthropic_sse_wrapper() + + if not isinstance(result, ResponsesAPIResponse): + raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") + + return _ADAPTER.translate_response(result) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py new file mode 100644 index 00000000000..0e6268e82f3 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -0,0 +1,265 @@ +# What is this? +## Translates OpenAI call to Anthropic `/v1/messages` format +import json +import traceback +from collections import deque +from typing import Any, AsyncIterator, Dict + +from litellm import verbose_logger +from litellm._uuid import uuid + + +class AnthropicResponsesStreamWrapper: + """ + Wraps a Responses API streaming iterator and re-emits events in Anthropic SSE format. + + Responses API event flow (relevant subset): + response.created -> message_start + response.output_item.added -> content_block_start (if message/function_call) + response.output_text.delta -> content_block_delta (text_delta) + response.reasoning_summary_text.delta -> content_block_delta (thinking_delta) + response.function_call_arguments.delta -> content_block_delta (input_json_delta) + response.output_item.done -> content_block_stop + response.completed -> message_delta + message_stop + """ + + def __init__( + self, + responses_stream: Any, + model: str, + ) -> None: + self.responses_stream = responses_stream + self.model = model + self._message_id: str = f"msg_{uuid.uuid4()}" + self._current_block_index: int = -1 + # Map item_id -> content_block_index so we can stop the right block later + self._item_id_to_block_index: Dict[str, int] = {} + # Track open function_call items by item_id so we can emit tool_use start + self._pending_tool_ids: Dict[str, str] = {} # item_id -> call_id / name accumulator + self._sent_message_start = False + self._sent_message_stop = False + self._chunk_queue: deque = deque() + + def _make_message_start(self) -> Dict[str, Any]: + return { + "type": "message_start", + "message": { + "id": self._message_id, + "type": "message", + "role": "assistant", + "content": [], + "model": self.model, + "stop_reason": None, + "stop_sequence": None, + "usage": { + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + }, + } + + def _next_block_index(self) -> int: + self._current_block_index += 1 + return self._current_block_index + + def _process_event(self, event: Any) -> None: + """Convert one Responses API event into zero or more Anthropic chunks queued for emission.""" + event_type = getattr(event, "type", None) + if event_type is None and isinstance(event, dict): + event_type = event.get("type") + + if event_type is None: + return + + # ---- message_start ---- + if event_type == "response.created": + self._sent_message_start = True + self._chunk_queue.append(self._make_message_start()) + return + + # ---- content_block_start for a new output message item ---- + if event_type == "response.output_item.added": + item = getattr(event, "item", None) or (event.get("item") if isinstance(event, dict) else None) + if item is None: + return + item_type = getattr(item, "type", None) or (item.get("type") if isinstance(item, dict) else None) + item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) + + if item_type == "message": + block_idx = self._next_block_index() + if item_id: + self._item_id_to_block_index[item_id] = block_idx + self._chunk_queue.append({ + "type": "content_block_start", + "index": block_idx, + "content_block": {"type": "text", "text": ""}, + }) + elif item_type == "function_call": + call_id = getattr(item, "call_id", None) or (item.get("call_id") if isinstance(item, dict) else None) or "" + name = getattr(item, "name", None) or (item.get("name") if isinstance(item, dict) else None) or "" + block_idx = self._next_block_index() + if item_id: + self._item_id_to_block_index[item_id] = block_idx + self._pending_tool_ids[item_id] = call_id + self._chunk_queue.append({ + "type": "content_block_start", + "index": block_idx, + "content_block": { + "type": "tool_use", + "id": call_id, + "name": name, + "input": {}, + }, + }) + elif item_type == "reasoning": + block_idx = self._next_block_index() + if item_id: + self._item_id_to_block_index[item_id] = block_idx + self._chunk_queue.append({ + "type": "content_block_start", + "index": block_idx, + "content_block": {"type": "thinking", "thinking": ""}, + }) + return + + # ---- text delta ---- + if event_type == "response.output_text.delta": + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "text_delta", "text": delta}, + }) + return + + # ---- reasoning summary text delta ---- + if event_type == "response.reasoning_summary_text.delta": + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "thinking_delta", "thinking": delta}, + }) + return + + # ---- function call arguments delta ---- + if event_type == "response.function_call_arguments.delta": + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "input_json_delta", "partial_json": delta}, + }) + return + + # ---- output item done -> content_block_stop ---- + if event_type == "response.output_item.done": + item = getattr(event, "item", None) or (event.get("item") if isinstance(event, dict) else None) + item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) if item else None + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_stop", + "index": block_idx, + }) + return + + # ---- response completed -> message_delta + message_stop ---- + if event_type in ("response.completed", "response.failed", "response.incomplete"): + response_obj = getattr(event, "response", None) or (event.get("response") if isinstance(event, dict) else None) + stop_reason = "end_turn" + input_tokens = 0 + output_tokens = 0 + cache_creation_tokens = 0 + cache_read_tokens = 0 + + if response_obj is not None: + status = getattr(response_obj, "status", None) + if status == "incomplete": + stop_reason = "max_tokens" + usage = getattr(response_obj, "usage", None) + if usage is not None: + input_tokens = getattr(usage, "input_tokens", 0) or 0 + output_tokens = getattr(usage, "output_tokens", 0) or 0 + cache_creation_tokens = getattr(usage, "input_tokens_details", None) + cache_read_tokens = getattr(usage, "output_tokens_details", None) + # Prefer direct cache fields if present + cache_creation_tokens = getattr(usage, "cache_creation_input_tokens", 0) or 0 + cache_read_tokens = getattr(usage, "cache_read_input_tokens", 0) or 0 + + # Check if tool_use was in the output to override stop_reason + if response_obj is not None: + output = getattr(response_obj, "output", []) or [] + for out_item in output: + out_type = getattr(out_item, "type", None) or (out_item.get("type") if isinstance(out_item, dict) else None) + if out_type == "function_call": + stop_reason = "tool_use" + break + + usage_delta: Dict[str, Any] = { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + } + if cache_creation_tokens: + usage_delta["cache_creation_input_tokens"] = cache_creation_tokens + if cache_read_tokens: + usage_delta["cache_read_input_tokens"] = cache_read_tokens + + self._chunk_queue.append({ + "type": "message_delta", + "delta": {"stop_reason": stop_reason, "stop_sequence": None}, + "usage": usage_delta, + }) + self._chunk_queue.append({"type": "message_stop"}) + self._sent_message_stop = True + return + + def __aiter__(self) -> "AnthropicResponsesStreamWrapper": + return self + + async def __anext__(self) -> Dict[str, Any]: + # Return any queued chunks first + if self._chunk_queue: + return self._chunk_queue.popleft() + + # Emit message_start if not yet done (fallback if response.created wasn't fired) + if not self._sent_message_start: + self._sent_message_start = True + self._chunk_queue.append(self._make_message_start()) + return self._chunk_queue.popleft() + + # Consume the upstream stream + try: + async for event in self.responses_stream: + self._process_event(event) + if self._chunk_queue: + return self._chunk_queue.popleft() + except StopAsyncIteration: + pass + except Exception as e: + verbose_logger.error( + f"AnthropicResponsesStreamWrapper error: {e}\n{traceback.format_exc()}" + ) + + # Drain any remaining queued chunks + if self._chunk_queue: + return self._chunk_queue.popleft() + + raise StopAsyncIteration + + async def async_anthropic_sse_wrapper(self) -> AsyncIterator[bytes]: + """Yield SSE-encoded bytes for each Anthropic event chunk.""" + async for chunk in self: + if isinstance(chunk, dict): + event_type: str = str(chunk.get("type", "message")) + payload = f"event: {event_type}\ndata: {json.dumps(chunk)}\n\n" + yield payload.encode() + else: + yield chunk diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py new file mode 100644 index 00000000000..a428e8f4e8f --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -0,0 +1,407 @@ +""" +Transformation layer: Anthropic /v1/messages <-> OpenAI Responses API. + +This module owns all format conversions for the direct v1/messages -> Responses API +path used for OpenAI and Azure models. +""" + +import json +from typing import Any, Dict, List, Optional, Union, cast + +from litellm.types.llms.anthropic import ( + AllAnthropicToolsValues, + AnthopicMessagesAssistantMessageParam, + AnthropicFinishReason, + AnthropicMessagesRequest, + AnthropicMessagesToolChoice, + AnthropicMessagesUserMessageParam, + AnthropicResponseContentBlockText, + AnthropicResponseContentBlockThinking, + AnthropicResponseContentBlockToolUse, +) +from litellm.types.llms.anthropic_messages.anthropic_response import ( + AnthropicMessagesResponse, + AnthropicUsage, +) +from litellm.types.llms.openai import ResponsesAPIResponse + + +class LiteLLMAnthropicToResponsesAPIAdapter: + """ + Converts Anthropic /v1/messages requests to OpenAI Responses API format and + converts Responses API responses back to Anthropic format. + """ + + # ------------------------------------------------------------------ # + # Request translation: Anthropic -> Responses API # + # ------------------------------------------------------------------ # + + @staticmethod + def _translate_anthropic_image_source_to_url(source: dict) -> Optional[str]: + """Convert Anthropic image source to a URL string.""" + source_type = source.get("type") + if source_type == "base64": + media_type = source.get("media_type", "image/jpeg") + data = source.get("data", "") + return f"data:{media_type};base64,{data}" if data else None + elif source_type == "url": + return source.get("url") + return None + + def translate_messages_to_responses_input( + self, + messages: List[ + Union[ + AnthropicMessagesUserMessageParam, + AnthopicMessagesAssistantMessageParam, + ] + ], + ) -> List[Dict[str, Any]]: + """ + Convert Anthropic messages list to Responses API `input` items. + + Mapping: + user text -> message(role=user, input_text) + user image -> message(role=user, input_image) + user tool_result -> function_call_output + assistant text -> message(role=assistant, output_text) + assistant tool_use -> function_call + """ + input_items: List[Dict[str, Any]] = [] + + for m in messages: + role = m["role"] + content = m.get("content") + + if role == "user": + if isinstance(content, str): + input_items.append({ + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": content}], + }) + elif isinstance(content, list): + user_parts: List[Dict[str, Any]] = [] + for block in content: + if not isinstance(block, dict): + continue + btype = block.get("type") + if btype == "text": + user_parts.append({"type": "input_text", "text": block.get("text", "")}) + elif btype == "image": + url = self._translate_anthropic_image_source_to_url(block.get("source", {})) + if url: + user_parts.append({"type": "input_image", "image_url": url}) + elif btype == "tool_result": + tool_use_id = block.get("tool_use_id", "") + inner = block.get("content") + if inner is None: + output_text = "" + elif isinstance(inner, str): + output_text = inner + elif isinstance(inner, list): + parts = [ + c.get("text", "") + for c in inner + if isinstance(c, dict) and c.get("type") == "text" + ] + output_text = "\n".join(parts) + else: + output_text = str(inner) + # tool_result is a top-level item, not inside the message + input_items.append({ + "type": "function_call_output", + "call_id": tool_use_id, + "output": output_text, + }) + if user_parts: + input_items.append({ + "type": "message", + "role": "user", + "content": user_parts, + }) + + elif role == "assistant": + if isinstance(content, str): + input_items.append({ + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": content}], + }) + elif isinstance(content, list): + asst_parts: List[Dict[str, Any]] = [] + for block in content: + if not isinstance(block, dict): + continue + btype = block.get("type") + if btype == "text": + asst_parts.append({"type": "output_text", "text": block.get("text", "")}) + elif btype == "tool_use": + # tool_use becomes a top-level function_call item + input_items.append({ + "type": "function_call", + "call_id": block.get("id", ""), + "name": block.get("name", ""), + "arguments": json.dumps(block.get("input", {})), + }) + elif btype == "thinking": + thinking_text = block.get("thinking", "") + if thinking_text: + asst_parts.append({"type": "output_text", "text": thinking_text}) + if asst_parts: + input_items.append({ + "type": "message", + "role": "assistant", + "content": asst_parts, + }) + + return input_items + + def translate_tools_to_responses_api( + self, + tools: List[AllAnthropicToolsValues], + ) -> List[Dict[str, Any]]: + """Convert Anthropic tool definitions to Responses API function tools.""" + result: List[Dict[str, Any]] = [] + for tool in tools: + tool_dict = cast(Dict[str, Any], tool) + tool_type = tool_dict.get("type", "") + tool_name = tool_dict.get("name", "") + # web_search tool + if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search": + result.append({"type": "web_search_preview"}) + continue + func_tool: Dict[str, Any] = {"type": "function", "name": tool_name} + if "description" in tool_dict: + func_tool["description"] = tool_dict["description"] + if "input_schema" in tool_dict: + func_tool["parameters"] = tool_dict["input_schema"] + result.append(func_tool) + return result + + @staticmethod + def translate_tool_choice_to_responses_api( + tool_choice: AnthropicMessagesToolChoice, + ) -> Dict[str, Any]: + """Convert Anthropic tool_choice to Responses API tool_choice.""" + tc_type = tool_choice.get("type") + if tc_type == "any": + return {"type": "required"} + elif tc_type == "tool": + return {"type": "function", "name": tool_choice.get("name", "")} + return {"type": "auto"} + + @staticmethod + def translate_thinking_to_reasoning(thinking: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """ + Convert Anthropic thinking param to Responses API reasoning param. + + thinking.budget_tokens maps to reasoning effort: + >= 10000 -> high, >= 5000 -> medium, >= 2000 -> low, < 2000 -> minimal + """ + if not isinstance(thinking, dict) or thinking.get("type") != "enabled": + return None + budget = thinking.get("budget_tokens", 0) + if budget >= 10000: + effort = "high" + elif budget >= 5000: + effort = "medium" + elif budget >= 2000: + effort = "low" + else: + effort = "minimal" + return {"effort": effort, "summary": "detailed"} + + def translate_request( + self, + anthropic_request: AnthropicMessagesRequest, + ) -> Dict[str, Any]: + """ + Translate a full Anthropic /v1/messages request dict to + litellm.responses() / litellm.aresponses() kwargs. + """ + model: str = anthropic_request["model"] + messages_list = cast( + List[Union[AnthropicMessagesUserMessageParam, AnthopicMessagesAssistantMessageParam]], + anthropic_request["messages"], + ) + + responses_kwargs: Dict[str, Any] = { + "model": model, + "input": self.translate_messages_to_responses_input(messages_list), + } + + # system -> instructions + system = anthropic_request.get("system") + if system: + if isinstance(system, str): + responses_kwargs["instructions"] = system + elif isinstance(system, list): + text_parts = [ + b.get("text", "") + for b in system + if isinstance(b, dict) and b.get("type") == "text" + ] + responses_kwargs["instructions"] = "\n".join(filter(None, text_parts)) + + # max_tokens -> max_output_tokens + max_tokens = anthropic_request.get("max_tokens") + if max_tokens: + responses_kwargs["max_output_tokens"] = max_tokens + + # temperature / top_p passed through + if "temperature" in anthropic_request: + responses_kwargs["temperature"] = anthropic_request["temperature"] + if "top_p" in anthropic_request: + responses_kwargs["top_p"] = anthropic_request["top_p"] + + # tools + tools = anthropic_request.get("tools") + if tools: + responses_kwargs["tools"] = self.translate_tools_to_responses_api( + cast(List[AllAnthropicToolsValues], tools) + ) + + # tool_choice + tool_choice = anthropic_request.get("tool_choice") + if tool_choice: + responses_kwargs["tool_choice"] = self.translate_tool_choice_to_responses_api( + cast(AnthropicMessagesToolChoice, tool_choice) + ) + + # thinking -> reasoning + thinking = anthropic_request.get("thinking") + if isinstance(thinking, dict): + reasoning = self.translate_thinking_to_reasoning(thinking) + if reasoning: + responses_kwargs["reasoning"] = reasoning + + # output_format -> text format + output_format = anthropic_request.get("output_format") + if isinstance(output_format, dict) and output_format.get("type") == "json_schema": + schema = output_format.get("schema") + if schema: + responses_kwargs["text"] = { + "format": { + "type": "json_schema", + "name": "structured_output", + "schema": schema, + "strict": True, + } + } + + # metadata user_id -> user + metadata = anthropic_request.get("metadata") + if isinstance(metadata, dict) and "user_id" in metadata: + responses_kwargs["user"] = metadata["user_id"] + + return responses_kwargs + + # ------------------------------------------------------------------ # + # Response translation: Responses API -> Anthropic # + # ------------------------------------------------------------------ # + + def translate_response( + self, + response: ResponsesAPIResponse, + ) -> AnthropicMessagesResponse: + """ + Translate an OpenAI ResponsesAPIResponse to AnthropicMessagesResponse. + """ + from openai.types.responses import ( + ResponseFunctionToolCall, + ResponseOutputMessage, + ResponseReasoningItem, + ) + + from litellm.types.llms.openai import ResponseAPIUsage + + content: List[Dict[str, Any]] = [] + stop_reason: AnthropicFinishReason = "end_turn" + + for item in response.output: + if isinstance(item, ResponseReasoningItem): + for summary in item.summary: + text = getattr(summary, "text", "") + if text: + content.append( + AnthropicResponseContentBlockThinking( + type="thinking", + thinking=text, + signature=None, + ).model_dump() + ) + + elif isinstance(item, ResponseOutputMessage): + for part in item.content: + if getattr(part, "type", None) == "output_text": + content.append( + AnthropicResponseContentBlockText( + type="text", text=getattr(part, "text", "") + ).model_dump() + ) + + elif isinstance(item, ResponseFunctionToolCall): + try: + input_data = json.loads(item.arguments) if item.arguments else {} + except (json.JSONDecodeError, TypeError): + input_data = {} + content.append( + AnthropicResponseContentBlockToolUse( + type="tool_use", + id=item.call_id or item.id, + name=item.name, + input=input_data, + ).model_dump() + ) + stop_reason = "tool_use" + + elif isinstance(item, dict): + item_type = item.get("type") + if item_type == "message": + for part in item.get("content", []): + if isinstance(part, dict) and part.get("type") == "output_text": + content.append( + AnthropicResponseContentBlockText( + type="text", text=part.get("text", "") + ).model_dump() + ) + elif item_type == "function_call": + try: + input_data = json.loads(item.get("arguments", "{}")) + except (json.JSONDecodeError, TypeError): + input_data = {} + content.append( + AnthropicResponseContentBlockToolUse( + type="tool_use", + id=item.get("call_id") or item.get("id", ""), + name=item.get("name", ""), + input=input_data, + ).model_dump() + ) + stop_reason = "tool_use" + + # status -> stop_reason override + if response.status == "incomplete": + stop_reason = "max_tokens" + + # usage + raw_usage: Optional[ResponseAPIUsage] = response.usage + input_tokens = int(getattr(raw_usage, "input_tokens", 0) or 0) + output_tokens = int(getattr(raw_usage, "output_tokens", 0) or 0) + + anthropic_usage = AnthropicUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + ) + + return AnthropicMessagesResponse( + id=response.id, + type="message", + role="assistant", + model=response.model or "unknown-model", + stop_sequence=None, + usage=anthropic_usage, # type: ignore + content=content, # type: ignore + stop_reason=stop_reason, + ) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 77c74a7847e..c671d9b37b8 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -16,13 +16,14 @@ from litellm.types.utils import Delta, ModelResponse, StreamingChoices def test_anthropic_experimental_pass_through_messages_handler(): """ - Test that api key is passed to litellm.completion + Test that api key is passed to litellm.responses for OpenAI models. + OpenAI and Azure models are routed directly to the Responses API. """ from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, ) - with patch("litellm.completion", return_value="test-response") as mock_completion: + with patch("litellm.responses", return_value="test-response") as mock_responses: try: anthropic_messages_handler( max_tokens=100, @@ -32,19 +33,20 @@ def test_anthropic_experimental_pass_through_messages_handler(): ) except Exception as e: print(f"Error: {e}") - mock_completion.assert_called_once() - assert mock_completion.call_args.kwargs["api_key"] == "test-api-key" + mock_responses.assert_called_once() + assert mock_responses.call_args.kwargs["api_key"] == "test-api-key" def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_and_api_base_and_custom_values(): """ - Test that api key is passed to litellm.completion + Test that api key, api base, and extra kwargs are forwarded to litellm.responses for Azure models. + Azure models are routed directly to the Responses API. """ from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, ) - with patch("litellm.completion", return_value="test-response") as mock_completion: + with patch("litellm.responses", return_value="test-response") as mock_responses: try: anthropic_messages_handler( max_tokens=100, @@ -56,10 +58,10 @@ def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_an ) except Exception as e: print(f"Error: {e}") - mock_completion.assert_called_once() - assert mock_completion.call_args.kwargs["api_key"] == "test-api-key" - assert mock_completion.call_args.kwargs["api_base"] == "test-api-base" - assert mock_completion.call_args.kwargs["custom_key"] == "custom_value" + mock_responses.assert_called_once() + assert mock_responses.call_args.kwargs["api_key"] == "test-api-key" + assert mock_responses.call_args.kwargs["api_base"] == "test-api-base" + assert mock_responses.call_args.kwargs["custom_key"] == "custom_value" def test_anthropic_experimental_pass_through_messages_handler_custom_llm_provider(): @@ -143,19 +145,19 @@ async def test_bedrock_converse_budget_tokens_preserved(): assert thinking_param.get("budget_tokens") == 1024, f"thinking.budget_tokens should be 1024, but got {thinking_param.get('budget_tokens')}" -def test_openai_model_with_thinking_converts_to_reasoning_effort(): +def test_openai_model_with_thinking_converts_to_reasoning(): """ - Test that when using a non-Anthropic model (like OpenAI gpt-5.2) with thinking parameter, - the thinking is converted to reasoning_effort and NOT passed as thinking. - - This ensures we don't regress on issue #16052 where non-Anthropic models would fail - with UnsupportedParamsError when thinking was passed directly. + Test that when using an OpenAI model with thinking parameter, the thinking is + converted to a Responses API `reasoning` param (NOT passed as thinking). + + OpenAI models are routed directly to the Responses API, so we verify that + litellm.responses() is called with `reasoning` properly set. """ from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, ) - with patch("litellm.completion", return_value="test-response") as mock_completion: + with patch("litellm.responses", return_value="test-response") as mock_responses: try: anthropic_messages_handler( max_tokens=1024, @@ -170,20 +172,22 @@ def test_openai_model_with_thinking_converts_to_reasoning_effort(): except Exception as e: print(f"Error: {e}") - mock_completion.assert_called_once() - - call_kwargs = mock_completion.call_args.kwargs - - # Verify reasoning_effort is set (converted from thinking) - assert "reasoning_effort" in call_kwargs, "reasoning_effort should be passed to completion" + mock_responses.assert_called_once() - # reasoning_effort is transformed into a dict with effort and summary fields - expected_reasoning_effort = {"effort": "minimal", "summary": "detailed"} - assert call_kwargs["reasoning_effort"] == expected_reasoning_effort, \ - f"reasoning_effort should be {expected_reasoning_effort} for budget_tokens=1024, got {call_kwargs.get('reasoning_effort')}" + call_kwargs = mock_responses.call_args.kwargs - # Verify thinking is NOT passed (non-Claude model) - assert "thinking" not in call_kwargs, "thinking should NOT be passed for non-Claude models" + # Verify reasoning is set (converted from thinking) + assert "reasoning" in call_kwargs, "reasoning should be passed to litellm.responses" + + # budget_tokens=1024 -> effort="minimal" (< 2000 threshold) + expected_reasoning = {"effort": "minimal", "summary": "detailed"} + assert call_kwargs["reasoning"] == expected_reasoning, ( + f"reasoning should be {expected_reasoning} for budget_tokens=1024, " + f"got {call_kwargs.get('reasoning')}" + ) + + # Verify thinking is NOT passed directly to the Responses API + assert "thinking" not in call_kwargs, "thinking should NOT be passed directly to litellm.responses" class TestThinkingParameterTransformation: