mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-05 08:07:05 +00:00
Add v1 for anthropic responses transformation
This commit is contained in:
Sameer Kankute
parent
a635bfe39a
commit
3e6b253cd6
6 changed files with 958 additions and 51 deletions
|
|
@ -25,8 +25,18 @@ from litellm.types.router import GenericLiteLLMParams
|
|||
from litellm.utils import ProviderConfigManager, client
|
||||
|
||||
from ..adapters.handler import LiteLLMMessagesToCompletionTransformationHandler
|
||||
from ..responses_adapters.handler import LiteLLMMessagesToResponsesAPIHandler
|
||||
from .utils import AnthropicMessagesRequestUtils, mock_response
|
||||
|
||||
# Providers that are routed directly to the OpenAI Responses API instead of
|
||||
# going through chat/completions.
|
||||
_RESPONSES_API_PROVIDERS = frozenset({"openai", "azure", "azure_text"})
|
||||
|
||||
|
||||
def _should_route_to_responses_api(custom_llm_provider: Optional[str]) -> bool:
|
||||
"""Return True when the provider should use the Responses API path."""
|
||||
return custom_llm_provider in _RESPONSES_API_PROVIDERS
|
||||
|
||||
####### ENVIRONMENT VARIABLES ###################
|
||||
# Initialize any necessary instances or variables here
|
||||
base_llm_http_handler = BaseLLMHTTPHandler()
|
||||
|
|
@ -282,29 +292,34 @@ def anthropic_messages_handler(
|
|||
)
|
||||
)
|
||||
if anthropic_messages_provider_config is None:
|
||||
# Handle non-Anthropic models using the adapter
|
||||
return (
|
||||
LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
model=model,
|
||||
metadata=metadata,
|
||||
stop_sequences=stop_sequences,
|
||||
stream=stream,
|
||||
system=system,
|
||||
temperature=temperature,
|
||||
thinking=thinking,
|
||||
tool_choice=tool_choice,
|
||||
tools=tools,
|
||||
top_k=top_k,
|
||||
top_p=top_p,
|
||||
_is_async=is_async,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
**kwargs,
|
||||
# Route to Responses API for OpenAI / Azure, chat/completions for everything else.
|
||||
_shared_kwargs = dict(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
model=model,
|
||||
metadata=metadata,
|
||||
stop_sequences=stop_sequences,
|
||||
stream=stream,
|
||||
system=system,
|
||||
temperature=temperature,
|
||||
thinking=thinking,
|
||||
tool_choice=tool_choice,
|
||||
tools=tools,
|
||||
top_k=top_k,
|
||||
top_p=top_p,
|
||||
_is_async=is_async,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
**kwargs,
|
||||
)
|
||||
if _should_route_to_responses_api(custom_llm_provider):
|
||||
return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
|
||||
**_shared_kwargs
|
||||
)
|
||||
return LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler(
|
||||
**_shared_kwargs
|
||||
)
|
||||
|
||||
if custom_llm_provider is None:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
|
||||
|
||||
__all__ = ["LiteLLMAnthropicToResponsesAPIAdapter"]
|
||||
|
|
@ -0,0 +1,213 @@
|
|||
"""
|
||||
Handler for the Anthropic v1/messages -> OpenAI Responses API path.
|
||||
|
||||
Used when the target model is an OpenAI or Azure model.
|
||||
"""
|
||||
|
||||
from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union
|
||||
|
||||
import litellm
|
||||
from litellm.types.llms.anthropic import AnthropicMessagesRequest
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
)
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
|
||||
from .streaming_iterator import AnthropicResponsesStreamWrapper
|
||||
from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
|
||||
|
||||
_ADAPTER = LiteLLMAnthropicToResponsesAPIAdapter()
|
||||
|
||||
|
||||
def _build_responses_kwargs(
|
||||
*,
|
||||
max_tokens: int,
|
||||
messages: List[Dict],
|
||||
model: str,
|
||||
metadata: Optional[Dict] = None,
|
||||
stop_sequences: Optional[List[str]] = None,
|
||||
stream: Optional[bool] = False,
|
||||
system: Optional[str] = None,
|
||||
temperature: Optional[float] = None,
|
||||
thinking: Optional[Dict] = None,
|
||||
tool_choice: Optional[Dict] = None,
|
||||
tools: Optional[List[Dict]] = None,
|
||||
top_k: Optional[int] = None,
|
||||
top_p: Optional[float] = None,
|
||||
output_format: Optional[Dict] = None,
|
||||
extra_kwargs: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses().
|
||||
"""
|
||||
# Build a typed AnthropicMessagesRequest for the adapter
|
||||
request_data: Dict[str, Any] = {"model": model, "messages": messages, "max_tokens": max_tokens}
|
||||
if metadata:
|
||||
request_data["metadata"] = metadata
|
||||
if system:
|
||||
request_data["system"] = system
|
||||
if temperature is not None:
|
||||
request_data["temperature"] = temperature
|
||||
if thinking:
|
||||
request_data["thinking"] = thinking
|
||||
if tool_choice:
|
||||
request_data["tool_choice"] = tool_choice
|
||||
if tools:
|
||||
request_data["tools"] = tools
|
||||
if top_p is not None:
|
||||
request_data["top_p"] = top_p
|
||||
if output_format:
|
||||
request_data["output_format"] = output_format
|
||||
|
||||
anthropic_request = AnthropicMessagesRequest(**request_data)
|
||||
responses_kwargs = _ADAPTER.translate_request(anthropic_request)
|
||||
|
||||
if stream:
|
||||
responses_kwargs["stream"] = True
|
||||
|
||||
# Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.)
|
||||
excluded = {"anthropic_messages"}
|
||||
for key, value in (extra_kwargs or {}).items():
|
||||
if key == "litellm_logging_obj" and value is not None:
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
Logging as LiteLLMLoggingObject,
|
||||
)
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
if isinstance(value, LiteLLMLoggingObject):
|
||||
# Reclassify as acompletion so the success handler doesn't try to
|
||||
# validate the Responses API event as an AnthropicResponse.
|
||||
# (Mirrors the pattern used in LiteLLMMessagesToCompletionTransformationHandler.)
|
||||
setattr(value, "call_type", CallTypes.acompletion.value)
|
||||
responses_kwargs[key] = value
|
||||
elif key not in excluded and key not in responses_kwargs and value is not None:
|
||||
responses_kwargs[key] = value
|
||||
|
||||
return responses_kwargs
|
||||
|
||||
|
||||
class LiteLLMMessagesToResponsesAPIHandler:
|
||||
"""
|
||||
Handles Anthropic /v1/messages requests for OpenAI / Azure models by
|
||||
calling litellm.responses() / litellm.aresponses() directly and translating
|
||||
the response back to Anthropic format.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
async def async_anthropic_messages_handler(
|
||||
max_tokens: int,
|
||||
messages: List[Dict],
|
||||
model: str,
|
||||
metadata: Optional[Dict] = None,
|
||||
stop_sequences: Optional[List[str]] = None,
|
||||
stream: Optional[bool] = False,
|
||||
system: Optional[str] = None,
|
||||
temperature: Optional[float] = None,
|
||||
thinking: Optional[Dict] = None,
|
||||
tool_choice: Optional[Dict] = None,
|
||||
tools: Optional[List[Dict]] = None,
|
||||
top_k: Optional[int] = None,
|
||||
top_p: Optional[float] = None,
|
||||
output_format: Optional[Dict] = None,
|
||||
**kwargs,
|
||||
) -> Union[AnthropicMessagesResponse, AsyncIterator]:
|
||||
responses_kwargs = _build_responses_kwargs(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
model=model,
|
||||
metadata=metadata,
|
||||
stop_sequences=stop_sequences,
|
||||
stream=stream,
|
||||
system=system,
|
||||
temperature=temperature,
|
||||
thinking=thinking,
|
||||
tool_choice=tool_choice,
|
||||
tools=tools,
|
||||
top_k=top_k,
|
||||
top_p=top_p,
|
||||
output_format=output_format,
|
||||
extra_kwargs=kwargs,
|
||||
)
|
||||
|
||||
result = await litellm.aresponses(**responses_kwargs)
|
||||
|
||||
if stream:
|
||||
wrapper = AnthropicResponsesStreamWrapper(responses_stream=result, model=model)
|
||||
return wrapper.async_anthropic_sse_wrapper()
|
||||
|
||||
if not isinstance(result, ResponsesAPIResponse):
|
||||
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
|
||||
|
||||
return _ADAPTER.translate_response(result)
|
||||
|
||||
@staticmethod
|
||||
def anthropic_messages_handler(
|
||||
max_tokens: int,
|
||||
messages: List[Dict],
|
||||
model: str,
|
||||
metadata: Optional[Dict] = None,
|
||||
stop_sequences: Optional[List[str]] = None,
|
||||
stream: Optional[bool] = False,
|
||||
system: Optional[str] = None,
|
||||
temperature: Optional[float] = None,
|
||||
thinking: Optional[Dict] = None,
|
||||
tool_choice: Optional[Dict] = None,
|
||||
tools: Optional[List[Dict]] = None,
|
||||
top_k: Optional[int] = None,
|
||||
top_p: Optional[float] = None,
|
||||
output_format: Optional[Dict] = None,
|
||||
_is_async: bool = False,
|
||||
**kwargs,
|
||||
) -> Union[
|
||||
AnthropicMessagesResponse,
|
||||
AsyncIterator[Any],
|
||||
Coroutine[Any, Any, Union[AnthropicMessagesResponse, AsyncIterator[Any]]],
|
||||
]:
|
||||
if _is_async:
|
||||
return LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
model=model,
|
||||
metadata=metadata,
|
||||
stop_sequences=stop_sequences,
|
||||
stream=stream,
|
||||
system=system,
|
||||
temperature=temperature,
|
||||
thinking=thinking,
|
||||
tool_choice=tool_choice,
|
||||
tools=tools,
|
||||
top_k=top_k,
|
||||
top_p=top_p,
|
||||
output_format=output_format,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# Sync path
|
||||
responses_kwargs = _build_responses_kwargs(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
model=model,
|
||||
metadata=metadata,
|
||||
stop_sequences=stop_sequences,
|
||||
stream=stream,
|
||||
system=system,
|
||||
temperature=temperature,
|
||||
thinking=thinking,
|
||||
tool_choice=tool_choice,
|
||||
tools=tools,
|
||||
top_k=top_k,
|
||||
top_p=top_p,
|
||||
output_format=output_format,
|
||||
extra_kwargs=kwargs,
|
||||
)
|
||||
|
||||
result = litellm.responses(**responses_kwargs)
|
||||
|
||||
if stream:
|
||||
wrapper = AnthropicResponsesStreamWrapper(responses_stream=result, model=model)
|
||||
return wrapper.async_anthropic_sse_wrapper()
|
||||
|
||||
if not isinstance(result, ResponsesAPIResponse):
|
||||
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
|
||||
|
||||
return _ADAPTER.translate_response(result)
|
||||
|
|
@ -0,0 +1,265 @@
|
|||
# What is this?
|
||||
## Translates OpenAI call to Anthropic `/v1/messages` format
|
||||
import json
|
||||
import traceback
|
||||
from collections import deque
|
||||
from typing import Any, AsyncIterator, Dict
|
||||
|
||||
from litellm import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
|
||||
|
||||
class AnthropicResponsesStreamWrapper:
|
||||
"""
|
||||
Wraps a Responses API streaming iterator and re-emits events in Anthropic SSE format.
|
||||
|
||||
Responses API event flow (relevant subset):
|
||||
response.created -> message_start
|
||||
response.output_item.added -> content_block_start (if message/function_call)
|
||||
response.output_text.delta -> content_block_delta (text_delta)
|
||||
response.reasoning_summary_text.delta -> content_block_delta (thinking_delta)
|
||||
response.function_call_arguments.delta -> content_block_delta (input_json_delta)
|
||||
response.output_item.done -> content_block_stop
|
||||
response.completed -> message_delta + message_stop
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
responses_stream: Any,
|
||||
model: str,
|
||||
) -> None:
|
||||
self.responses_stream = responses_stream
|
||||
self.model = model
|
||||
self._message_id: str = f"msg_{uuid.uuid4()}"
|
||||
self._current_block_index: int = -1
|
||||
# Map item_id -> content_block_index so we can stop the right block later
|
||||
self._item_id_to_block_index: Dict[str, int] = {}
|
||||
# Track open function_call items by item_id so we can emit tool_use start
|
||||
self._pending_tool_ids: Dict[str, str] = {} # item_id -> call_id / name accumulator
|
||||
self._sent_message_start = False
|
||||
self._sent_message_stop = False
|
||||
self._chunk_queue: deque = deque()
|
||||
|
||||
def _make_message_start(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": self._message_id,
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [],
|
||||
"model": self.model,
|
||||
"stop_reason": None,
|
||||
"stop_sequence": None,
|
||||
"usage": {
|
||||
"input_tokens": 0,
|
||||
"output_tokens": 0,
|
||||
"cache_creation_input_tokens": 0,
|
||||
"cache_read_input_tokens": 0,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
def _next_block_index(self) -> int:
|
||||
self._current_block_index += 1
|
||||
return self._current_block_index
|
||||
|
||||
def _process_event(self, event: Any) -> None:
|
||||
"""Convert one Responses API event into zero or more Anthropic chunks queued for emission."""
|
||||
event_type = getattr(event, "type", None)
|
||||
if event_type is None and isinstance(event, dict):
|
||||
event_type = event.get("type")
|
||||
|
||||
if event_type is None:
|
||||
return
|
||||
|
||||
# ---- message_start ----
|
||||
if event_type == "response.created":
|
||||
self._sent_message_start = True
|
||||
self._chunk_queue.append(self._make_message_start())
|
||||
return
|
||||
|
||||
# ---- content_block_start for a new output message item ----
|
||||
if event_type == "response.output_item.added":
|
||||
item = getattr(event, "item", None) or (event.get("item") if isinstance(event, dict) else None)
|
||||
if item is None:
|
||||
return
|
||||
item_type = getattr(item, "type", None) or (item.get("type") if isinstance(item, dict) else None)
|
||||
item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None)
|
||||
|
||||
if item_type == "message":
|
||||
block_idx = self._next_block_index()
|
||||
if item_id:
|
||||
self._item_id_to_block_index[item_id] = block_idx
|
||||
self._chunk_queue.append({
|
||||
"type": "content_block_start",
|
||||
"index": block_idx,
|
||||
"content_block": {"type": "text", "text": ""},
|
||||
})
|
||||
elif item_type == "function_call":
|
||||
call_id = getattr(item, "call_id", None) or (item.get("call_id") if isinstance(item, dict) else None) or ""
|
||||
name = getattr(item, "name", None) or (item.get("name") if isinstance(item, dict) else None) or ""
|
||||
block_idx = self._next_block_index()
|
||||
if item_id:
|
||||
self._item_id_to_block_index[item_id] = block_idx
|
||||
self._pending_tool_ids[item_id] = call_id
|
||||
self._chunk_queue.append({
|
||||
"type": "content_block_start",
|
||||
"index": block_idx,
|
||||
"content_block": {
|
||||
"type": "tool_use",
|
||||
"id": call_id,
|
||||
"name": name,
|
||||
"input": {},
|
||||
},
|
||||
})
|
||||
elif item_type == "reasoning":
|
||||
block_idx = self._next_block_index()
|
||||
if item_id:
|
||||
self._item_id_to_block_index[item_id] = block_idx
|
||||
self._chunk_queue.append({
|
||||
"type": "content_block_start",
|
||||
"index": block_idx,
|
||||
"content_block": {"type": "thinking", "thinking": ""},
|
||||
})
|
||||
return
|
||||
|
||||
# ---- text delta ----
|
||||
if event_type == "response.output_text.delta":
|
||||
item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None)
|
||||
delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "")
|
||||
block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index
|
||||
self._chunk_queue.append({
|
||||
"type": "content_block_delta",
|
||||
"index": block_idx,
|
||||
"delta": {"type": "text_delta", "text": delta},
|
||||
})
|
||||
return
|
||||
|
||||
# ---- reasoning summary text delta ----
|
||||
if event_type == "response.reasoning_summary_text.delta":
|
||||
item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None)
|
||||
delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "")
|
||||
block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index
|
||||
self._chunk_queue.append({
|
||||
"type": "content_block_delta",
|
||||
"index": block_idx,
|
||||
"delta": {"type": "thinking_delta", "thinking": delta},
|
||||
})
|
||||
return
|
||||
|
||||
# ---- function call arguments delta ----
|
||||
if event_type == "response.function_call_arguments.delta":
|
||||
item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None)
|
||||
delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "")
|
||||
block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index
|
||||
self._chunk_queue.append({
|
||||
"type": "content_block_delta",
|
||||
"index": block_idx,
|
||||
"delta": {"type": "input_json_delta", "partial_json": delta},
|
||||
})
|
||||
return
|
||||
|
||||
# ---- output item done -> content_block_stop ----
|
||||
if event_type == "response.output_item.done":
|
||||
item = getattr(event, "item", None) or (event.get("item") if isinstance(event, dict) else None)
|
||||
item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) if item else None
|
||||
block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index
|
||||
self._chunk_queue.append({
|
||||
"type": "content_block_stop",
|
||||
"index": block_idx,
|
||||
})
|
||||
return
|
||||
|
||||
# ---- response completed -> message_delta + message_stop ----
|
||||
if event_type in ("response.completed", "response.failed", "response.incomplete"):
|
||||
response_obj = getattr(event, "response", None) or (event.get("response") if isinstance(event, dict) else None)
|
||||
stop_reason = "end_turn"
|
||||
input_tokens = 0
|
||||
output_tokens = 0
|
||||
cache_creation_tokens = 0
|
||||
cache_read_tokens = 0
|
||||
|
||||
if response_obj is not None:
|
||||
status = getattr(response_obj, "status", None)
|
||||
if status == "incomplete":
|
||||
stop_reason = "max_tokens"
|
||||
usage = getattr(response_obj, "usage", None)
|
||||
if usage is not None:
|
||||
input_tokens = getattr(usage, "input_tokens", 0) or 0
|
||||
output_tokens = getattr(usage, "output_tokens", 0) or 0
|
||||
cache_creation_tokens = getattr(usage, "input_tokens_details", None)
|
||||
cache_read_tokens = getattr(usage, "output_tokens_details", None)
|
||||
# Prefer direct cache fields if present
|
||||
cache_creation_tokens = getattr(usage, "cache_creation_input_tokens", 0) or 0
|
||||
cache_read_tokens = getattr(usage, "cache_read_input_tokens", 0) or 0
|
||||
|
||||
# Check if tool_use was in the output to override stop_reason
|
||||
if response_obj is not None:
|
||||
output = getattr(response_obj, "output", []) or []
|
||||
for out_item in output:
|
||||
out_type = getattr(out_item, "type", None) or (out_item.get("type") if isinstance(out_item, dict) else None)
|
||||
if out_type == "function_call":
|
||||
stop_reason = "tool_use"
|
||||
break
|
||||
|
||||
usage_delta: Dict[str, Any] = {
|
||||
"input_tokens": input_tokens,
|
||||
"output_tokens": output_tokens,
|
||||
}
|
||||
if cache_creation_tokens:
|
||||
usage_delta["cache_creation_input_tokens"] = cache_creation_tokens
|
||||
if cache_read_tokens:
|
||||
usage_delta["cache_read_input_tokens"] = cache_read_tokens
|
||||
|
||||
self._chunk_queue.append({
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": stop_reason, "stop_sequence": None},
|
||||
"usage": usage_delta,
|
||||
})
|
||||
self._chunk_queue.append({"type": "message_stop"})
|
||||
self._sent_message_stop = True
|
||||
return
|
||||
|
||||
def __aiter__(self) -> "AnthropicResponsesStreamWrapper":
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> Dict[str, Any]:
|
||||
# Return any queued chunks first
|
||||
if self._chunk_queue:
|
||||
return self._chunk_queue.popleft()
|
||||
|
||||
# Emit message_start if not yet done (fallback if response.created wasn't fired)
|
||||
if not self._sent_message_start:
|
||||
self._sent_message_start = True
|
||||
self._chunk_queue.append(self._make_message_start())
|
||||
return self._chunk_queue.popleft()
|
||||
|
||||
# Consume the upstream stream
|
||||
try:
|
||||
async for event in self.responses_stream:
|
||||
self._process_event(event)
|
||||
if self._chunk_queue:
|
||||
return self._chunk_queue.popleft()
|
||||
except StopAsyncIteration:
|
||||
pass
|
||||
except Exception as e:
|
||||
verbose_logger.error(
|
||||
f"AnthropicResponsesStreamWrapper error: {e}\n{traceback.format_exc()}"
|
||||
)
|
||||
|
||||
# Drain any remaining queued chunks
|
||||
if self._chunk_queue:
|
||||
return self._chunk_queue.popleft()
|
||||
|
||||
raise StopAsyncIteration
|
||||
|
||||
async def async_anthropic_sse_wrapper(self) -> AsyncIterator[bytes]:
|
||||
"""Yield SSE-encoded bytes for each Anthropic event chunk."""
|
||||
async for chunk in self:
|
||||
if isinstance(chunk, dict):
|
||||
event_type: str = str(chunk.get("type", "message"))
|
||||
payload = f"event: {event_type}\ndata: {json.dumps(chunk)}\n\n"
|
||||
yield payload.encode()
|
||||
else:
|
||||
yield chunk
|
||||
|
|
@ -0,0 +1,407 @@
|
|||
"""
|
||||
Transformation layer: Anthropic /v1/messages <-> OpenAI Responses API.
|
||||
|
||||
This module owns all format conversions for the direct v1/messages -> Responses API
|
||||
path used for OpenAI and Azure models.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional, Union, cast
|
||||
|
||||
from litellm.types.llms.anthropic import (
|
||||
AllAnthropicToolsValues,
|
||||
AnthopicMessagesAssistantMessageParam,
|
||||
AnthropicFinishReason,
|
||||
AnthropicMessagesRequest,
|
||||
AnthropicMessagesToolChoice,
|
||||
AnthropicMessagesUserMessageParam,
|
||||
AnthropicResponseContentBlockText,
|
||||
AnthropicResponseContentBlockThinking,
|
||||
AnthropicResponseContentBlockToolUse,
|
||||
)
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
AnthropicUsage,
|
||||
)
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
|
||||
|
||||
class LiteLLMAnthropicToResponsesAPIAdapter:
|
||||
"""
|
||||
Converts Anthropic /v1/messages requests to OpenAI Responses API format and
|
||||
converts Responses API responses back to Anthropic format.
|
||||
"""
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Request translation: Anthropic -> Responses API #
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
@staticmethod
|
||||
def _translate_anthropic_image_source_to_url(source: dict) -> Optional[str]:
|
||||
"""Convert Anthropic image source to a URL string."""
|
||||
source_type = source.get("type")
|
||||
if source_type == "base64":
|
||||
media_type = source.get("media_type", "image/jpeg")
|
||||
data = source.get("data", "")
|
||||
return f"data:{media_type};base64,{data}" if data else None
|
||||
elif source_type == "url":
|
||||
return source.get("url")
|
||||
return None
|
||||
|
||||
def translate_messages_to_responses_input(
|
||||
self,
|
||||
messages: List[
|
||||
Union[
|
||||
AnthropicMessagesUserMessageParam,
|
||||
AnthopicMessagesAssistantMessageParam,
|
||||
]
|
||||
],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Convert Anthropic messages list to Responses API `input` items.
|
||||
|
||||
Mapping:
|
||||
user text -> message(role=user, input_text)
|
||||
user image -> message(role=user, input_image)
|
||||
user tool_result -> function_call_output
|
||||
assistant text -> message(role=assistant, output_text)
|
||||
assistant tool_use -> function_call
|
||||
"""
|
||||
input_items: List[Dict[str, Any]] = []
|
||||
|
||||
for m in messages:
|
||||
role = m["role"]
|
||||
content = m.get("content")
|
||||
|
||||
if role == "user":
|
||||
if isinstance(content, str):
|
||||
input_items.append({
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": content}],
|
||||
})
|
||||
elif isinstance(content, list):
|
||||
user_parts: List[Dict[str, Any]] = []
|
||||
for block in content:
|
||||
if not isinstance(block, dict):
|
||||
continue
|
||||
btype = block.get("type")
|
||||
if btype == "text":
|
||||
user_parts.append({"type": "input_text", "text": block.get("text", "")})
|
||||
elif btype == "image":
|
||||
url = self._translate_anthropic_image_source_to_url(block.get("source", {}))
|
||||
if url:
|
||||
user_parts.append({"type": "input_image", "image_url": url})
|
||||
elif btype == "tool_result":
|
||||
tool_use_id = block.get("tool_use_id", "")
|
||||
inner = block.get("content")
|
||||
if inner is None:
|
||||
output_text = ""
|
||||
elif isinstance(inner, str):
|
||||
output_text = inner
|
||||
elif isinstance(inner, list):
|
||||
parts = [
|
||||
c.get("text", "")
|
||||
for c in inner
|
||||
if isinstance(c, dict) and c.get("type") == "text"
|
||||
]
|
||||
output_text = "\n".join(parts)
|
||||
else:
|
||||
output_text = str(inner)
|
||||
# tool_result is a top-level item, not inside the message
|
||||
input_items.append({
|
||||
"type": "function_call_output",
|
||||
"call_id": tool_use_id,
|
||||
"output": output_text,
|
||||
})
|
||||
if user_parts:
|
||||
input_items.append({
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": user_parts,
|
||||
})
|
||||
|
||||
elif role == "assistant":
|
||||
if isinstance(content, str):
|
||||
input_items.append({
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": content}],
|
||||
})
|
||||
elif isinstance(content, list):
|
||||
asst_parts: List[Dict[str, Any]] = []
|
||||
for block in content:
|
||||
if not isinstance(block, dict):
|
||||
continue
|
||||
btype = block.get("type")
|
||||
if btype == "text":
|
||||
asst_parts.append({"type": "output_text", "text": block.get("text", "")})
|
||||
elif btype == "tool_use":
|
||||
# tool_use becomes a top-level function_call item
|
||||
input_items.append({
|
||||
"type": "function_call",
|
||||
"call_id": block.get("id", ""),
|
||||
"name": block.get("name", ""),
|
||||
"arguments": json.dumps(block.get("input", {})),
|
||||
})
|
||||
elif btype == "thinking":
|
||||
thinking_text = block.get("thinking", "")
|
||||
if thinking_text:
|
||||
asst_parts.append({"type": "output_text", "text": thinking_text})
|
||||
if asst_parts:
|
||||
input_items.append({
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": asst_parts,
|
||||
})
|
||||
|
||||
return input_items
|
||||
|
||||
def translate_tools_to_responses_api(
|
||||
self,
|
||||
tools: List[AllAnthropicToolsValues],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Convert Anthropic tool definitions to Responses API function tools."""
|
||||
result: List[Dict[str, Any]] = []
|
||||
for tool in tools:
|
||||
tool_dict = cast(Dict[str, Any], tool)
|
||||
tool_type = tool_dict.get("type", "")
|
||||
tool_name = tool_dict.get("name", "")
|
||||
# web_search tool
|
||||
if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search":
|
||||
result.append({"type": "web_search_preview"})
|
||||
continue
|
||||
func_tool: Dict[str, Any] = {"type": "function", "name": tool_name}
|
||||
if "description" in tool_dict:
|
||||
func_tool["description"] = tool_dict["description"]
|
||||
if "input_schema" in tool_dict:
|
||||
func_tool["parameters"] = tool_dict["input_schema"]
|
||||
result.append(func_tool)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def translate_tool_choice_to_responses_api(
|
||||
tool_choice: AnthropicMessagesToolChoice,
|
||||
) -> Dict[str, Any]:
|
||||
"""Convert Anthropic tool_choice to Responses API tool_choice."""
|
||||
tc_type = tool_choice.get("type")
|
||||
if tc_type == "any":
|
||||
return {"type": "required"}
|
||||
elif tc_type == "tool":
|
||||
return {"type": "function", "name": tool_choice.get("name", "")}
|
||||
return {"type": "auto"}
|
||||
|
||||
@staticmethod
|
||||
def translate_thinking_to_reasoning(thinking: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Convert Anthropic thinking param to Responses API reasoning param.
|
||||
|
||||
thinking.budget_tokens maps to reasoning effort:
|
||||
>= 10000 -> high, >= 5000 -> medium, >= 2000 -> low, < 2000 -> minimal
|
||||
"""
|
||||
if not isinstance(thinking, dict) or thinking.get("type") != "enabled":
|
||||
return None
|
||||
budget = thinking.get("budget_tokens", 0)
|
||||
if budget >= 10000:
|
||||
effort = "high"
|
||||
elif budget >= 5000:
|
||||
effort = "medium"
|
||||
elif budget >= 2000:
|
||||
effort = "low"
|
||||
else:
|
||||
effort = "minimal"
|
||||
return {"effort": effort, "summary": "detailed"}
|
||||
|
||||
def translate_request(
|
||||
self,
|
||||
anthropic_request: AnthropicMessagesRequest,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Translate a full Anthropic /v1/messages request dict to
|
||||
litellm.responses() / litellm.aresponses() kwargs.
|
||||
"""
|
||||
model: str = anthropic_request["model"]
|
||||
messages_list = cast(
|
||||
List[Union[AnthropicMessagesUserMessageParam, AnthopicMessagesAssistantMessageParam]],
|
||||
anthropic_request["messages"],
|
||||
)
|
||||
|
||||
responses_kwargs: Dict[str, Any] = {
|
||||
"model": model,
|
||||
"input": self.translate_messages_to_responses_input(messages_list),
|
||||
}
|
||||
|
||||
# system -> instructions
|
||||
system = anthropic_request.get("system")
|
||||
if system:
|
||||
if isinstance(system, str):
|
||||
responses_kwargs["instructions"] = system
|
||||
elif isinstance(system, list):
|
||||
text_parts = [
|
||||
b.get("text", "")
|
||||
for b in system
|
||||
if isinstance(b, dict) and b.get("type") == "text"
|
||||
]
|
||||
responses_kwargs["instructions"] = "\n".join(filter(None, text_parts))
|
||||
|
||||
# max_tokens -> max_output_tokens
|
||||
max_tokens = anthropic_request.get("max_tokens")
|
||||
if max_tokens:
|
||||
responses_kwargs["max_output_tokens"] = max_tokens
|
||||
|
||||
# temperature / top_p passed through
|
||||
if "temperature" in anthropic_request:
|
||||
responses_kwargs["temperature"] = anthropic_request["temperature"]
|
||||
if "top_p" in anthropic_request:
|
||||
responses_kwargs["top_p"] = anthropic_request["top_p"]
|
||||
|
||||
# tools
|
||||
tools = anthropic_request.get("tools")
|
||||
if tools:
|
||||
responses_kwargs["tools"] = self.translate_tools_to_responses_api(
|
||||
cast(List[AllAnthropicToolsValues], tools)
|
||||
)
|
||||
|
||||
# tool_choice
|
||||
tool_choice = anthropic_request.get("tool_choice")
|
||||
if tool_choice:
|
||||
responses_kwargs["tool_choice"] = self.translate_tool_choice_to_responses_api(
|
||||
cast(AnthropicMessagesToolChoice, tool_choice)
|
||||
)
|
||||
|
||||
# thinking -> reasoning
|
||||
thinking = anthropic_request.get("thinking")
|
||||
if isinstance(thinking, dict):
|
||||
reasoning = self.translate_thinking_to_reasoning(thinking)
|
||||
if reasoning:
|
||||
responses_kwargs["reasoning"] = reasoning
|
||||
|
||||
# output_format -> text format
|
||||
output_format = anthropic_request.get("output_format")
|
||||
if isinstance(output_format, dict) and output_format.get("type") == "json_schema":
|
||||
schema = output_format.get("schema")
|
||||
if schema:
|
||||
responses_kwargs["text"] = {
|
||||
"format": {
|
||||
"type": "json_schema",
|
||||
"name": "structured_output",
|
||||
"schema": schema,
|
||||
"strict": True,
|
||||
}
|
||||
}
|
||||
|
||||
# metadata user_id -> user
|
||||
metadata = anthropic_request.get("metadata")
|
||||
if isinstance(metadata, dict) and "user_id" in metadata:
|
||||
responses_kwargs["user"] = metadata["user_id"]
|
||||
|
||||
return responses_kwargs
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Response translation: Responses API -> Anthropic #
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def translate_response(
|
||||
self,
|
||||
response: ResponsesAPIResponse,
|
||||
) -> AnthropicMessagesResponse:
|
||||
"""
|
||||
Translate an OpenAI ResponsesAPIResponse to AnthropicMessagesResponse.
|
||||
"""
|
||||
from openai.types.responses import (
|
||||
ResponseFunctionToolCall,
|
||||
ResponseOutputMessage,
|
||||
ResponseReasoningItem,
|
||||
)
|
||||
|
||||
from litellm.types.llms.openai import ResponseAPIUsage
|
||||
|
||||
content: List[Dict[str, Any]] = []
|
||||
stop_reason: AnthropicFinishReason = "end_turn"
|
||||
|
||||
for item in response.output:
|
||||
if isinstance(item, ResponseReasoningItem):
|
||||
for summary in item.summary:
|
||||
text = getattr(summary, "text", "")
|
||||
if text:
|
||||
content.append(
|
||||
AnthropicResponseContentBlockThinking(
|
||||
type="thinking",
|
||||
thinking=text,
|
||||
signature=None,
|
||||
).model_dump()
|
||||
)
|
||||
|
||||
elif isinstance(item, ResponseOutputMessage):
|
||||
for part in item.content:
|
||||
if getattr(part, "type", None) == "output_text":
|
||||
content.append(
|
||||
AnthropicResponseContentBlockText(
|
||||
type="text", text=getattr(part, "text", "")
|
||||
).model_dump()
|
||||
)
|
||||
|
||||
elif isinstance(item, ResponseFunctionToolCall):
|
||||
try:
|
||||
input_data = json.loads(item.arguments) if item.arguments else {}
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
input_data = {}
|
||||
content.append(
|
||||
AnthropicResponseContentBlockToolUse(
|
||||
type="tool_use",
|
||||
id=item.call_id or item.id,
|
||||
name=item.name,
|
||||
input=input_data,
|
||||
).model_dump()
|
||||
)
|
||||
stop_reason = "tool_use"
|
||||
|
||||
elif isinstance(item, dict):
|
||||
item_type = item.get("type")
|
||||
if item_type == "message":
|
||||
for part in item.get("content", []):
|
||||
if isinstance(part, dict) and part.get("type") == "output_text":
|
||||
content.append(
|
||||
AnthropicResponseContentBlockText(
|
||||
type="text", text=part.get("text", "")
|
||||
).model_dump()
|
||||
)
|
||||
elif item_type == "function_call":
|
||||
try:
|
||||
input_data = json.loads(item.get("arguments", "{}"))
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
input_data = {}
|
||||
content.append(
|
||||
AnthropicResponseContentBlockToolUse(
|
||||
type="tool_use",
|
||||
id=item.get("call_id") or item.get("id", ""),
|
||||
name=item.get("name", ""),
|
||||
input=input_data,
|
||||
).model_dump()
|
||||
)
|
||||
stop_reason = "tool_use"
|
||||
|
||||
# status -> stop_reason override
|
||||
if response.status == "incomplete":
|
||||
stop_reason = "max_tokens"
|
||||
|
||||
# usage
|
||||
raw_usage: Optional[ResponseAPIUsage] = response.usage
|
||||
input_tokens = int(getattr(raw_usage, "input_tokens", 0) or 0)
|
||||
output_tokens = int(getattr(raw_usage, "output_tokens", 0) or 0)
|
||||
|
||||
anthropic_usage = AnthropicUsage(
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
)
|
||||
|
||||
return AnthropicMessagesResponse(
|
||||
id=response.id,
|
||||
type="message",
|
||||
role="assistant",
|
||||
model=response.model or "unknown-model",
|
||||
stop_sequence=None,
|
||||
usage=anthropic_usage, # type: ignore
|
||||
content=content, # type: ignore
|
||||
stop_reason=stop_reason,
|
||||
)
|
||||
|
|
@ -16,13 +16,14 @@ from litellm.types.utils import Delta, ModelResponse, StreamingChoices
|
|||
|
||||
def test_anthropic_experimental_pass_through_messages_handler():
|
||||
"""
|
||||
Test that api key is passed to litellm.completion
|
||||
Test that api key is passed to litellm.responses for OpenAI models.
|
||||
OpenAI and Azure models are routed directly to the Responses API.
|
||||
"""
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
|
||||
anthropic_messages_handler,
|
||||
)
|
||||
|
||||
with patch("litellm.completion", return_value="test-response") as mock_completion:
|
||||
with patch("litellm.responses", return_value="test-response") as mock_responses:
|
||||
try:
|
||||
anthropic_messages_handler(
|
||||
max_tokens=100,
|
||||
|
|
@ -32,19 +33,20 @@ def test_anthropic_experimental_pass_through_messages_handler():
|
|||
)
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
mock_completion.assert_called_once()
|
||||
assert mock_completion.call_args.kwargs["api_key"] == "test-api-key"
|
||||
mock_responses.assert_called_once()
|
||||
assert mock_responses.call_args.kwargs["api_key"] == "test-api-key"
|
||||
|
||||
|
||||
def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_and_api_base_and_custom_values():
|
||||
"""
|
||||
Test that api key is passed to litellm.completion
|
||||
Test that api key, api base, and extra kwargs are forwarded to litellm.responses for Azure models.
|
||||
Azure models are routed directly to the Responses API.
|
||||
"""
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
|
||||
anthropic_messages_handler,
|
||||
)
|
||||
|
||||
with patch("litellm.completion", return_value="test-response") as mock_completion:
|
||||
with patch("litellm.responses", return_value="test-response") as mock_responses:
|
||||
try:
|
||||
anthropic_messages_handler(
|
||||
max_tokens=100,
|
||||
|
|
@ -56,10 +58,10 @@ def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_an
|
|||
)
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
mock_completion.assert_called_once()
|
||||
assert mock_completion.call_args.kwargs["api_key"] == "test-api-key"
|
||||
assert mock_completion.call_args.kwargs["api_base"] == "test-api-base"
|
||||
assert mock_completion.call_args.kwargs["custom_key"] == "custom_value"
|
||||
mock_responses.assert_called_once()
|
||||
assert mock_responses.call_args.kwargs["api_key"] == "test-api-key"
|
||||
assert mock_responses.call_args.kwargs["api_base"] == "test-api-base"
|
||||
assert mock_responses.call_args.kwargs["custom_key"] == "custom_value"
|
||||
|
||||
|
||||
def test_anthropic_experimental_pass_through_messages_handler_custom_llm_provider():
|
||||
|
|
@ -143,19 +145,19 @@ async def test_bedrock_converse_budget_tokens_preserved():
|
|||
assert thinking_param.get("budget_tokens") == 1024, f"thinking.budget_tokens should be 1024, but got {thinking_param.get('budget_tokens')}"
|
||||
|
||||
|
||||
def test_openai_model_with_thinking_converts_to_reasoning_effort():
|
||||
def test_openai_model_with_thinking_converts_to_reasoning():
|
||||
"""
|
||||
Test that when using a non-Anthropic model (like OpenAI gpt-5.2) with thinking parameter,
|
||||
the thinking is converted to reasoning_effort and NOT passed as thinking.
|
||||
|
||||
This ensures we don't regress on issue #16052 where non-Anthropic models would fail
|
||||
with UnsupportedParamsError when thinking was passed directly.
|
||||
Test that when using an OpenAI model with thinking parameter, the thinking is
|
||||
converted to a Responses API `reasoning` param (NOT passed as thinking).
|
||||
|
||||
OpenAI models are routed directly to the Responses API, so we verify that
|
||||
litellm.responses() is called with `reasoning` properly set.
|
||||
"""
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
|
||||
anthropic_messages_handler,
|
||||
)
|
||||
|
||||
with patch("litellm.completion", return_value="test-response") as mock_completion:
|
||||
with patch("litellm.responses", return_value="test-response") as mock_responses:
|
||||
try:
|
||||
anthropic_messages_handler(
|
||||
max_tokens=1024,
|
||||
|
|
@ -170,20 +172,22 @@ def test_openai_model_with_thinking_converts_to_reasoning_effort():
|
|||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
|
||||
mock_completion.assert_called_once()
|
||||
|
||||
call_kwargs = mock_completion.call_args.kwargs
|
||||
|
||||
# Verify reasoning_effort is set (converted from thinking)
|
||||
assert "reasoning_effort" in call_kwargs, "reasoning_effort should be passed to completion"
|
||||
mock_responses.assert_called_once()
|
||||
|
||||
# reasoning_effort is transformed into a dict with effort and summary fields
|
||||
expected_reasoning_effort = {"effort": "minimal", "summary": "detailed"}
|
||||
assert call_kwargs["reasoning_effort"] == expected_reasoning_effort, \
|
||||
f"reasoning_effort should be {expected_reasoning_effort} for budget_tokens=1024, got {call_kwargs.get('reasoning_effort')}"
|
||||
call_kwargs = mock_responses.call_args.kwargs
|
||||
|
||||
# Verify thinking is NOT passed (non-Claude model)
|
||||
assert "thinking" not in call_kwargs, "thinking should NOT be passed for non-Claude models"
|
||||
# Verify reasoning is set (converted from thinking)
|
||||
assert "reasoning" in call_kwargs, "reasoning should be passed to litellm.responses"
|
||||
|
||||
# budget_tokens=1024 -> effort="minimal" (< 2000 threshold)
|
||||
expected_reasoning = {"effort": "minimal", "summary": "detailed"}
|
||||
assert call_kwargs["reasoning"] == expected_reasoning, (
|
||||
f"reasoning should be {expected_reasoning} for budget_tokens=1024, "
|
||||
f"got {call_kwargs.get('reasoning')}"
|
||||
)
|
||||
|
||||
# Verify thinking is NOT passed directly to the Responses API
|
||||
assert "thinking" not in call_kwargs, "thinking should NOT be passed directly to litellm.responses"
|
||||
|
||||
|
||||
class TestThinkingParameterTransformation:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue