Merge remote-tracking branch 'origin/main' into litellm_same_provider_hop_keeps_encrypted_reasoning
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled

This commit is contained in:
mateo-berri 2026-10-29 03:45:02 -07:00
commit 14beda3b6e
145 changed files with 9287 additions and 474 deletions

View file

@ -236,6 +236,7 @@ jobs:
tests/unit/llms/azure_ai
tests/unit/llms/base_llm
tests/unit/llms/baseten
tests/unit/llms/bigmodel
tests/unit/llms/black_forest_labs
tests/unit/llms/brave
tests/unit/llms/bytez

View file

@ -30,7 +30,7 @@ from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, cast
from typing import TYPE_CHECKING, Final
from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
ManagedObjectCreatorAttribution,
@ -290,12 +290,14 @@ class CheckResponsesCost:
sees provider env vars, so it fails for every deployment whose credentials
live in the config; the row then never leaves ``queued``.
"""
if not via_router:
return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
router_response = await self.llm_router.aget_responses(
response_id=response_id, litellm_metadata=litellm_metadata
retrieved: Final[object] = (
await self.llm_router.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
if via_router
else await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
)
return cast(ResponsesAPIResponse, router_response)
if not isinstance(retrieved, ResponsesAPIResponse):
raise TypeError(f"Retrieving {response_id} did not return a response object")
return retrieved
async def _fetch_unbilled(self, target: _PollTarget) -> ResponsesAPIResponse:
return await self._get_response(

63
examples/bigmodel.md Normal file
View file

@ -0,0 +1,63 @@
# BigModel (Zhipu AI)
Use `bigmodel/` for the domestic BigModel platform and `zai/` for international Z.AI. Each platform uses its own credentials and default URL
```yaml
model_list:
- model_name: glm
litellm_params:
model: bigmodel/glm-5.3
api_key: os.environ/BIGMODEL_API_KEY
api_base: https://open.bigmodel.cn
```
One model configuration supports all three native APIs. The provider selects the upstream endpoint from the requested API
| Proxy endpoint | BigModel endpoint |
| --- | --- |
| `/v1/chat/completions` | `/api/paas/v4/chat/completions` |
| `/v1/responses` | `/api/v1/responses` |
| `/v1/messages` | `/api/anthropic/v1/messages` |
`api_base` is optional and defaults to `https://open.bigmodel.cn`. Set `BIGMODEL_API_BASE` to override the default, or pass an explicit `api_base` to override the environment value. Existing API-specific base URLs and complete endpoint URLs are normalized, preserving a custom proxy's path prefix
A GLM Coding Plan key uses `https://open.bigmodel.cn/api/coding/paas/v4` as its `api_base`, since Coding Plan and standard keys do not work on each other's endpoints. Chat Completions then calls the Coding Plan endpoint, and Responses and Messages keep the endpoints in the table above
The Admin UI lists BigModel (Zhipu AI), displays its official logo, and provides a required password API Key field and an optional API Base field with the default URL. Authentication uses an explicit `api_key` or `BIGMODEL_API_KEY`
```python
import litellm
chat = litellm.completion(
model="bigmodel/glm-5.3",
messages=[{"role": "user", "content": "Hello"}],
thinking={"type": "enabled"},
reasoning_effort="low",
)
response = litellm.responses(
model="bigmodel/glm-5.3",
input="Hello",
reasoning={"effort": "low"},
store=True,
)
async def messages():
return await litellm.anthropic.messages.acreate(
model="bigmodel/glm-5.3",
max_tokens=1024,
messages=[{"role": "user", "content": "Hello"}],
thinking={"type": "adaptive"},
output_config={"effort": "low"},
)
```
Each API retains its native response format and streaming events. The native Messages SDK entry point is asynchronous, as is the proxy's `/v1/messages` route
GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX use always-on thinking. GLM-5.3 supports `low`, `high`, and `max` reasoning effort. Messages preserves native `thinking`, `output_config`, and `reasoning_effort` parameters
Set `store=True` when creating a Responses request if you need continuation, retrieval, or deletion. Use the response ID returned by LiteLLM. BigModel documents a seven-day retention period and does not support cancel, compact, native WebSocket, or file_search
The model catalog and packaged backup include GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX. Prices are expressed in USD/token using a static 7.0 CNY/USD conversion, with the source and verification date in each entry's metadata. Use existing `litellm_params` cost fields for contract-specific deployment prices
API references: [Chat Completions](https://docs.bigmodel.cn/cn/guide/develop/openai/introduction), [Responses](https://docs.bigmodel.cn/cn/guide/develop/responses/introduction), [Messages](https://docs.bigmodel.cn/cn/guide/develop/claude/introduction), and [pricing](https://docs.bigmodel.cn/cn/guide/start/pricing), checked on 2026-10-27

View file

@ -619,6 +619,8 @@ vertex_openai_models: Set = set()
vertex_minimax_models: Set = set()
vertex_moonshot_models: Set = set()
vertex_zai_models: Set = set()
vertex_qwen_models: Final[Set[str]] = set()
vertex_ai_models: Final[Set[str]] = set()
ai21_models: Set = set()
ai21_chat_models: Set = set()
nlp_cloud_models: Set = set()
@ -635,6 +637,7 @@ watsonx_models: Set = set()
gemini_models: Set = set()
xai_models: Set = set()
zai_models: Set = set()
bigmodel_models: Final[set[str]] = set() # mutable-ok: provider registry is populated and extended at runtime
deepseek_models: Set = set()
tencent_models: Set = set()
runwayml_models: Set = set()
@ -833,6 +836,10 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
elif value.get("litellm_provider") == "vertex_ai-zai_models":
key = key.replace("vertex_ai/", "")
vertex_zai_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-qwen_models":
vertex_qwen_models.add(key.removeprefix("vertex_ai/"))
elif value.get("litellm_provider") == "vertex_ai":
vertex_ai_models.add(key.removeprefix("vertex_ai/"))
elif value.get("litellm_provider") == "ai21":
if value.get("mode") == "chat":
ai21_chat_models.add(key)
@ -877,6 +884,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
xai_models.add(key)
elif value.get("litellm_provider") == "zai":
zai_models.add(key)
elif value.get("litellm_provider") == "bigmodel":
bigmodel_models.add(key)
elif value.get("litellm_provider") == "fal_ai":
fal_ai_models.add(key)
elif value.get("litellm_provider") == "deepseek":
@ -1107,6 +1116,7 @@ model_list = list(
| text_completion_inception_models
| xai_models
| zai_models
| bigmodel_models
| fal_ai_models
| deepseek_models
| modelscope_models
@ -1204,7 +1214,16 @@ def _build_models_by_provider() -> dict:
| vertex_deepseek_models
| vertex_minimax_models
| vertex_moonshot_models
| vertex_zai_models,
| vertex_zai_models
| vertex_qwen_models
| vertex_mistral_models
| vertex_llama3_models
| vertex_ai_ai21_models
| vertex_openai_models
| vertex_ai_image_models
| vertex_ai_video_models
| vertex_embedding_models
| vertex_ai_models,
"ai21": ai21_models | ai21_chat_models,
"bedrock": bedrock_models | bedrock_converse_models,
"petals": petals_models,
@ -1221,6 +1240,7 @@ def _build_models_by_provider() -> dict:
"text-completion-inception": text_completion_inception_models,
"xai": xai_models,
"zai": zai_models,
"bigmodel": bigmodel_models,
"fal_ai": fal_ai_models,
"deepseek": deepseek_models,
"tencent": tencent_models,

View file

@ -260,6 +260,7 @@ class ResponsesToCompletionBridgeHandler:
structured_output=is_structured_output_request(
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
),
replays_openai_history=custom_llm_provider == "openai",
)
streamwrapper: Final = CustomStreamWrapper(
completion_stream=completion_stream,
@ -363,6 +364,7 @@ class ResponsesToCompletionBridgeHandler:
structured_output=is_structured_output_request(
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
),
replays_openai_history=custom_llm_provider == "openai",
)
streamwrapper: Final = CustomStreamWrapper(
completion_stream=completion_stream,

View file

@ -0,0 +1,432 @@
import hashlib
import json
from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass
from itertools import accumulate, chain, pairwise
from types import MappingProxyType
from typing import Annotated, Final, Literal, TypeAlias
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
REPLAY_FIELD: Final = "openai_responses_replay"
PHASE_FIELD: Final = "phase"
Phase: TypeAlias = Literal["commentary", "final_answer"]
class _Frozen(BaseModel):
model_config = ConfigDict(frozen=True)
class _OutputText(_Frozen):
type: Literal["output_text"]
text: str
class _OutputMessage(_Frozen):
type: Literal["message"]
role: Literal["assistant"]
content: tuple[_OutputText, ...]
phase: Phase | None = None
class _OutputReasoning(_Frozen):
type: Literal["reasoning"]
id: str
class _OutputToolCall(_Frozen):
type: Literal["function_call", "custom_tool_call"]
call_id: str
id: str | None = None
class _OutputToolSearch(_Frozen):
type: Literal["tool_search_call", "tool_search_output"]
_OutputItem: TypeAlias = Annotated[
_OutputMessage | _OutputReasoning | _OutputToolCall | _OutputToolSearch, Field(discriminator="type")
]
_OUTPUT_ITEM: Final[TypeAdapter[_OutputItem]] = TypeAdapter(_OutputItem)
class _AnyMessage(_Frozen):
type: Literal["message"]
class _MessageEntry(_Frozen):
type: Literal["message"]
parts: tuple[Annotated[int, Field(ge=0)], ...]
phase: Phase | None = None
class _IndexedEntry(_Frozen):
type: Literal["reasoning", "function_call", "custom_tool_call", "tool_search_call", "tool_search_output"]
index: Annotated[int, Field(ge=0)]
_Entry: TypeAlias = Annotated[_MessageEntry | _IndexedEntry, Field(discriminator="type")]
class _Replay(_Frozen):
version: Literal[1]
items: tuple[_Entry, ...]
checksum: str
class _Namespaced(_Frozen):
namespace: str | None = None
class _ToolFunction(_Frozen):
name: str = ""
arguments: str = ""
provider_specific_fields: _Namespaced | None = None
class _ToolCall(_Frozen):
id: str
function: _ToolFunction
provider_specific_fields: _Namespaced | None = None
class _SummaryText(_Frozen):
type: str = "summary_text"
text: str = ""
class _Reasoning(_Frozen):
id: str | None = None
encrypted_content: str | None = None
summary: tuple[_SummaryText, ...] | None = None
class _ProviderFields(_Frozen):
phase: Phase | None = None
tool_search_items: tuple[Mapping[str, object], ...] | None = None
openai_responses_replay: object = None
class _ChatMessage(_Frozen):
role: str | None = None
content: str | None = None
tool_calls: tuple[_ToolCall, ...] | None = None
reasoning_items: tuple[_Reasoning, ...] | None = None
phase: Phase | None = None
provider_specific_fields: _ProviderFields | None = None
class _PhaseFields(_Frozen):
phase: Phase | None = None
class _RequestedPhaseFields(_Frozen):
phase: object = None
class _RequestedPhase(_Frozen):
phase: object = None
provider_specific_fields: _RequestedPhaseFields | None = None
@dataclass(frozen=True, slots=True)
class UnknownPhase:
value: object
_PHASE: Final[TypeAdapter[Phase]] = TypeAdapter(Phase)
class _RetainedFields(_PhaseFields):
openai_responses_replay: object = None
@dataclass(frozen=True, slots=True)
class ReplayedTurn:
items: tuple[Mapping[str, object], ...]
custom_call_ids: frozenset[str]
def _as_json(value: object) -> object:
return value.model_dump() if isinstance(value, BaseModel) else value
def _parsed_output_item(item: object) -> _OutputItem | None:
try:
return _OUTPUT_ITEM.validate_python(_as_json(item))
except ValidationError:
return None
def _is_message(item: object) -> bool:
try:
_AnyMessage.model_validate(_as_json(item))
except ValidationError:
return False
return True
def _chat_message(message: object) -> _ChatMessage | None:
try:
return _ChatMessage.model_validate(_as_json(message))
except ValidationError:
return None
def requested_phase(message: object) -> Phase | UnknownPhase | None:
try:
requested: Final = _RequestedPhase.model_validate(_as_json(message))
except ValidationError:
return None
fields: Final = requested.provider_specific_fields
value: Final = requested.phase or (fields.phase if fields else None)
if value is None:
return None
try:
return _PHASE.validate_python(value)
except ValidationError:
return UnknownPhase(value)
def reasoning_input_item(
item_id: str | None, summary: Sequence[object] | None, encrypted_content: str | None
) -> Mapping[str, object]:
return {
"type": "reasoning",
**({"id": item_id} if item_id else {}),
"summary": list(summary or ()),
**({"encrypted_content": encrypted_content} if encrypted_content else {}),
}
def retained_replay_fields(provider_specific_fields: object) -> Mapping[str, object] | None:
try:
retained: Final = _RetainedFields.model_validate(provider_specific_fields)
except ValidationError:
return None
return retained.model_dump(exclude_none=True) or None
def output_message_text(output_items: Sequence[object], *, first_message_only: bool) -> str:
parsed: Final = map(_parsed_output_item, output_items)
messages: Final = tuple(item for item in parsed if isinstance(item, _OutputMessage))
return "".join(_texts(messages[:1] if first_message_only else messages))
def _tool_search_entries(message: _ChatMessage) -> tuple[Mapping[str, object], ...]:
fields: Final = message.provider_specific_fields
items: Final = (fields.tool_search_items if fields else None) or ()
return tuple(item for item in items if item.get("type") in ("tool_search_call", "tool_search_output"))
def _checksum(message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry]) -> str:
fields: Final = message.provider_specific_fields
bound: Final = {
"content": message.content or "",
"tool_calls": [[call.id, call.function.name, call.function.arguments] for call in message.tool_calls or ()],
"reasoning": [
[item.id, item.encrypted_content, [part.text for part in item.summary or ()]]
for item in message.reasoning_items or ()
],
"tool_search_items": list((fields.tool_search_items if fields else None) or ()),
"phase": message.phase or (fields.phase if fields else None),
"items": [entry.model_dump(mode="json", exclude_none=True) for entry in entries],
}
encoded: Final = json.dumps(bound, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
return hashlib.sha256(encoded.encode("utf-8", "surrogatepass")).hexdigest()
def _kind(item: _OutputItem) -> str:
match item:
case _OutputToolCall():
return "tool_call"
case _OutputToolSearch():
return "tool_search"
case _OutputMessage() | _OutputReasoning():
return item.type
def _entry(items: Sequence[_OutputItem], position: int) -> _MessageEntry | _IndexedEntry:
item: Final = items[position]
if isinstance(item, _OutputMessage):
return _MessageEntry(type="message", parts=tuple(len(part.text) for part in item.content), phase=item.phase)
ordinal: Final = sum(1 for earlier in items[:position] if _kind(earlier) == _kind(item))
return _IndexedEntry(type=item.type, index=ordinal)
def _texts(items: Sequence[_OutputItem]) -> Iterator[str]:
for item in items:
if isinstance(item, _OutputMessage):
yield from (part.text for part in item.content)
def _matches(items: Sequence[_OutputItem], message: _ChatMessage) -> bool:
reasoning_ids: Final = tuple(item.id for item in items if isinstance(item, _OutputReasoning))
calls: Final = tuple(item for item in items if isinstance(item, _OutputToolCall))
tool_calls: Final = message.tool_calls or ()
searches: Final = sum(1 for item in items if isinstance(item, _OutputToolSearch))
return (
"".join(_texts(items)) == (message.content or "")
and reasoning_ids == tuple(item.id for item in message.reasoning_items or ())
and len(calls) == len(tool_calls)
and all(chat.id in (call.call_id, call.id) for call, chat in zip(calls, tool_calls))
and searches == len(_tool_search_entries(message))
)
def _content_phase(
output_items: Sequence[object], parsed: Sequence[_OutputItem | None], *, first_message_only: bool
) -> Phase | None:
messages: Final = tuple(item for raw, item in zip(output_items, parsed) if _is_message(raw))
if not messages:
return None
content_message: Final = messages[0] if first_message_only else messages[-1]
return content_message.phase if isinstance(content_message, _OutputMessage) else None
def _with_phase(message: _ChatMessage, phase: Phase | None) -> _ChatMessage:
if phase is None:
return message
fields: Final = (message.provider_specific_fields or _ProviderFields()).model_copy(update={PHASE_FIELD: phase})
return message.model_copy(update={"provider_specific_fields": fields})
def replay_fields(
output_items: Sequence[object], message: object, *, completed: bool, first_message_only: bool
) -> Mapping[str, object]:
parsed: Final = tuple(map(_parsed_output_item, output_items))
phase: Final = _content_phase(output_items, parsed, first_message_only=first_message_only)
phase_fields: Final[Mapping[str, object]] = MappingProxyType({PHASE_FIELD: phase} if phase else {})
items: Final = tuple(item for item in parsed if item is not None)
parsed_message: Final = _chat_message(message)
chat: Final = _with_phase(parsed_message, phase) if parsed_message is not None else None
if not completed or not items or len(items) != len(parsed) or chat is None or not _matches(items, chat):
return phase_fields
entries: Final = tuple(_entry(items, position) for position in range(len(items)))
replay: Final = _Replay(version=1, items=entries, checksum=_checksum(chat, entries))
return MappingProxyType({**phase_fields, REPLAY_FIELD: replay.model_dump(mode="json", exclude_none=True)})
def _stored_replay(message: _ChatMessage) -> _Replay | None:
fields: Final = message.provider_specific_fields
if fields is None or fields.openai_responses_replay is None:
return None
try:
return _Replay.model_validate(fields.openai_responses_replay)
except ValidationError:
return None
def _message_texts(entries: Sequence[_MessageEntry], content: str) -> tuple[tuple[str, ...], ...]:
lengths: Final = tuple(chain.from_iterable(entry.parts for entry in entries))
bounds: Final = tuple(accumulate(lengths, initial=0))
parts: Final = tuple(content[start:end] for start, end in pairwise(bounds))
groups: Final = tuple(accumulate((len(entry.parts) for entry in entries), initial=0))
return tuple(parts[start:end] for start, end in pairwise(groups))
def _message_item(texts: Sequence[str], phase: Phase | None) -> Mapping[str, object]:
return {
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": text} for text in texts],
**({PHASE_FIELD: phase} if phase else {}),
}
def _function_call_item(call: _ToolCall) -> Mapping[str, object]:
namespaced: Final = call.provider_specific_fields or call.function.provider_specific_fields
namespace: Final = namespaced.namespace if namespaced else None
return {
"type": "function_call",
"call_id": call.id,
"name": call.function.name,
"arguments": call.function.arguments,
**({"namespace": namespace} if namespace else {}),
}
def _custom_tool_call_item(call: _ToolCall) -> Mapping[str, object]:
return {
"type": "custom_tool_call",
"call_id": call.id,
"name": call.function.name,
"input": call.function.arguments,
}
def _reasoning_item(item: _Reasoning) -> Mapping[str, object]:
summary: Final = tuple(part.model_dump() for part in item.summary or ())
return reasoning_input_item(item.id, summary, item.encrypted_content)
@dataclass(frozen=True, slots=True)
class _Resolved:
message: _ChatMessage
texts: Mapping[int, tuple[str, ...]]
tool_searches: tuple[Mapping[str, object], ...]
def _input_item(resolved: _Resolved, position: int, entry: _MessageEntry | _IndexedEntry) -> Mapping[str, object]:
if isinstance(entry, _MessageEntry):
return _message_item(resolved.texts[position], entry.phase)
tool_calls: Final = resolved.message.tool_calls or ()
match entry.type:
case "reasoning":
return _reasoning_item((resolved.message.reasoning_items or ())[entry.index])
case "function_call":
return _function_call_item(tool_calls[entry.index])
case "custom_tool_call":
return _custom_tool_call_item(tool_calls[entry.index])
case "tool_search_call" | "tool_search_output":
return dict(resolved.tool_searches[entry.index])
def _fits(message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry]) -> bool:
tool_calls: Final = len(message.tool_calls or ())
tool_searches: Final = len(_tool_search_entries(message))
sizes: Final = MappingProxyType(
{
"reasoning": len(message.reasoning_items or ()),
"function_call": tool_calls,
"custom_tool_call": tool_calls,
"tool_search_call": tool_searches,
"tool_search_output": tool_searches,
}
)
parts: Final = chain.from_iterable(entry.parts for entry in entries if isinstance(entry, _MessageEntry))
indexed: Final = (entry for entry in entries if isinstance(entry, _IndexedEntry))
return sum(parts) == len(message.content or "") and all(entry.index < sizes[entry.type] for entry in indexed)
def _resolve(
message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry], replay_tool_search: bool
) -> ReplayedTurn | None:
indexed: Final = tuple(entry for entry in entries if isinstance(entry, _IndexedEntry))
if not replay_tool_search and any(entry.type in ("tool_search_call", "tool_search_output") for entry in indexed):
return None
messages: Final = tuple(entry for entry in entries if isinstance(entry, _MessageEntry))
calls: Final = tuple(entry for entry in indexed if entry.type in ("function_call", "custom_tool_call"))
texts: Final = _message_texts(messages, message.content or "")
positions: Final = tuple(position for position, entry in enumerate(entries) if isinstance(entry, _MessageEntry))
resolved: Final = _Resolved(
message=message,
texts=MappingProxyType(dict(zip(positions, texts, strict=True))),
tool_searches=_tool_search_entries(message),
)
tool_calls: Final = message.tool_calls or ()
return ReplayedTurn(
items=tuple(_input_item(resolved, position, entry) for position, entry in enumerate(entries)),
custom_call_ids=frozenset(tool_calls[entry.index].id for entry in calls if entry.type == "custom_tool_call"),
)
def replay_items(message: object, *, replay_tool_search: bool) -> ReplayedTurn | None:
chat: Final = _chat_message(message)
stored: Final = _stored_replay(chat) if chat is not None and chat.role == "assistant" else None
if chat is None or stored is None or stored.checksum != _checksum(chat, stored.items):
return None
if not _fits(chat, stored.items):
return None
return _resolve(chat, stored.items, replay_tool_search)

View file

@ -24,9 +24,19 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
import litellm
from litellm import ModelResponse
from litellm._logging import verbose_logger
from litellm.completion_extras.litellm_responses_transformation.history import (
PHASE_FIELD,
UnknownPhase,
output_message_text,
reasoning_input_item,
replay_fields,
replay_items,
requested_phase,
)
from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
prompt_cache_breakpoint_for_wire,
responses_reasoning_items_from_thinking_blocks,
with_prompt_cache_breakpoint,
)
@ -52,7 +62,6 @@ from litellm.types.llms.openai import (
ChatCompletionToolCallChunk,
ChatCompletionToolCallFunctionChunk,
ChatCompletionToolParamFunctionChunk,
PromptCacheBreakpoint,
Reasoning,
ResponsesAPIOptionalRequestParams,
ResponsesAPIResponse,
@ -490,7 +499,7 @@ def _fold_choices_into_one(
choices: Sequence["Choices"],
tool_calls: Sequence[Mapping[str, object]],
trailing_reasoning_content: str | None,
trailing_reasoning_item: _BuiltReasoningItem | None,
trailing_reasoning_items: Sequence[_BuiltReasoningItem],
tool_search_items: Sequence[Mapping[str, object]],
structured_output: bool = False,
) -> "Choices":
@ -509,7 +518,7 @@ def _fold_choices_into_one(
reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content)
reasoning_items: Final = (
*chain.from_iterable(_message_reasoning_items(m) for m in messages),
*(() if trailing_reasoning_item is None else (trailing_reasoning_item,)),
*trailing_reasoning_items,
)
return Choices(
message=Message(
@ -526,6 +535,61 @@ def _fold_choices_into_one(
)
def _with_replay_fields(
choice: "Choices", output_items: Sequence[object], *, completed: bool, structured_output: bool
) -> "Choices":
fields: Final = {
**(choice.message.provider_specific_fields or {}),
**replay_fields(output_items, choice.message, completed=completed, first_message_only=structured_output),
}
message: Final = choice.message.model_copy(update={"provider_specific_fields": fields or None})
return choice.model_copy(update={"message": message})
_STREAMED_OUTPUT_ITEMS: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter(
tuple[Mapping[str, object], ...]
)
def completed_stream_history(
output_payload: object,
tool_search_fields: Mapping[str, object] | None,
*,
completed: bool,
structured_output: bool,
) -> dict[str, object] | None: # mutable-ok: Delta.provider_specific_fields is typed as a dict
items: Final = _STREAMED_OUTPUT_ITEMS.validate_python(output_payload)
calls: Final = tuple(item for item in items if item.get("type") in ("function_call", "custom_tool_call"))
streamed_message: Final = {
"role": "assistant",
"content": output_message_text(items, first_message_only=structured_output),
"tool_calls": [tool_call_dict_from_output_item(call, index) for index, call in enumerate(calls)],
"reasoning_items": list(_reasoning_items_from_output_items(items)),
"provider_specific_fields": tool_search_fields,
}
fields: Final = {
**(tool_search_fields or {}),
**replay_fields(items, streamed_message, completed=completed, first_message_only=structured_output),
}
return fields or None
def _assistant_phase_field(msg: object, replay_openai_history: bool, drop_params: bool) -> Mapping[str, object]:
match requested_phase(msg) if replay_openai_history else None:
case None:
return {}
case UnknownPhase(value=value) if drop_params:
verbose_logger.debug("Chat provider: dropping assistant phase=%r under drop_params", value)
return {}
case UnknownPhase(value=value):
raise litellm.UnsupportedParamsError(
message=f"phase must be 'commentary' or 'final_answer', got {value!r}. "
"To drop it, set `litellm.drop_params=True` or pass `drop_params=True`."
)
case phase:
return {PHASE_FIELD: phase}
def _responses_event_type(chunk: Mapping[str, object]) -> object:
event_type: Final = chunk.get("type")
return event_type.value if isinstance(event_type, ResponsesAPIStreamEvents) else event_type
@ -567,19 +631,6 @@ def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Li
return "length"
_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint)
def _prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
if marker is None or not drop_params:
return marker
try:
return _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
except ValidationError:
verbose_logger.debug("Chat provider: dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
return None
def _input_file_from_file_value(file_value: object) -> dict[str, object]:
if not isinstance(file_value, dict):
return {"type": "input_file"}
@ -667,15 +718,7 @@ def _reasoning_item_to_response_input(
An item without an id is sent without one: the Responses API accepts that and
verifies the encrypted content on its own, while it rejects any id it did not mint.
"""
item_id: Final = r_item.get("id")
encrypted_content: Final = r_item.get("encrypted_content")
return {
"type": "reasoning",
**({"id": item_id} if item_id else {}),
# summary is always required by the Responses API, even when empty
"summary": r_item.get("summary") or [],
**({"encrypted_content": encrypted_content} if encrypted_content else {}),
}
return dict(reasoning_input_item(r_item.get("id"), r_item.get("summary"), r_item.get("encrypted_content")))
def _requested_defer_loading(tool: Mapping[str, object], function_tool: Mapping[str, object]) -> object:
@ -804,12 +847,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
keep_prompt_cache_breakpoints: bool = False,
replay_tool_search: bool = False,
include_namespace: bool = False,
replay_openai_history: bool = False,
) -> tuple[list[object], str | None]:
converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input(
messages,
drop_params=drop_params,
replay_tool_search=replay_tool_search,
include_namespace=include_namespace,
replay_openai_history=replay_openai_history,
)
return (
converted_input_items
@ -825,10 +870,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
drop_params: bool = False,
replay_tool_search: bool = False,
include_namespace: bool = False,
replay_openai_history: bool = False,
) -> tuple[list[object], str | None]:
input_items: Final[list[object]] = []
instructions: str | None = None
custom_tool_call_ids: Final = frozenset(
replayed_turns: Final = tuple(
replay_items(msg, replay_tool_search=replay_tool_search) if replay_openai_history else None
for msg in messages
)
replayed_custom_call_ids: Final = frozenset(
chain.from_iterable(turn.custom_call_ids for turn in replayed_turns if turn is not None)
)
custom_tool_call_ids: Final = replayed_custom_call_ids | frozenset(
tool_call["id"]
for msg in messages
if msg.get("role") == "assistant" and isinstance(msg.get("tool_calls"), list)
@ -849,7 +902,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
tool_calls = msg.get("tool_calls")
tool_call_id = msg.get("tool_call_id")
if role == "system":
if (replayed_turn := replayed_turns[index]) is not None:
input_items.extend(replayed_turn.items)
elif role == "system":
# Extract system message as instructions
if isinstance(content, str) and index < leading_system_count:
if instructions:
@ -915,6 +970,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
"content": self._convert_content_to_responses_format(
content, "assistant", drop_params=drop_params
),
**_assistant_phase_field(msg, replay_openai_history, drop_params),
}
)
for tool_call in tool_calls:
@ -955,6 +1011,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
"content": self._convert_content_to_responses_format(
content, cast(str, role), drop_params=drop_params
),
**_assistant_phase_field(msg, replay_openai_history and role == "assistant", drop_params),
}
)
elif role == "assistant":
@ -1081,6 +1138,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast
include_namespace=True,
replay_openai_history=provider == "openai",
)
# OpenAI's Responses API rejects an empty input. For a system-only
# request, carry the system message as a system-role input item instead
@ -1165,6 +1223,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
output_items: Sequence[object],
handle_raw_dict_callback: Callable[..., tuple["Choices | None", int]] | None = None,
structured_output: bool = False,
replay_openai_history: bool = False,
completed: bool = False,
) -> list["Choices"]:
"""
Convert Responses API output items to chat completion choices.
@ -1174,6 +1234,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
handle_raw_dict_callback: Optional callback for handling raw dict items
structured_output: The chat request carried a json response_format, so only the
first message item becomes content
replay_openai_history: Record the output's item sequence for the next turn
completed: The response finished, so its item sequence is whole
Returns:
List of Choices objects
@ -1196,7 +1258,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
choices: Final[list[Choices]] = []
index = 0
reasoning_content: str | None = None
pending_reasoning_item: _BuiltReasoningItem | None = None
pending_reasoning_items: tuple[_BuiltReasoningItem, ...] = ()
# Collect all tool calls to put them in a single choice
# (Chat Completions API expects all tool calls in one message)
@ -1205,12 +1267,19 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
for item in output_items:
if isinstance(item, ResponseReasoningItem):
pending_reasoning_item = _build_reasoning_item(
item_id=item.id,
encrypted_content=getattr(item, "encrypted_content", None),
summary_raw=item.summary,
pending_reasoning_items = (
*(pending_reasoning_items if replay_openai_history else ()),
_build_reasoning_item(
item_id=item.id,
encrypted_content=getattr(item, "encrypted_content", None),
summary_raw=item.summary,
),
)
reasoning_content = " ".join(
s["text"]
for s in chain.from_iterable(built["summary"] for built in pending_reasoning_items)
if s.get("text")
)
reasoning_content = " ".join(s["text"] for s in pending_reasoning_item["summary"] if s.get("text"))
elif isinstance(item, ResponseOutputMessage):
for content in item.content:
@ -1227,7 +1296,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
annotations=annotations,
reasoning_items=cast(
list[ChatCompletionReasoningItem] | None,
([pending_reasoning_item] if pending_reasoning_item is not None else None),
(list(pending_reasoning_items) or None),
),
)
@ -1240,7 +1309,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
reasoning_content = None # flush
pending_reasoning_item = None # flush
pending_reasoning_items = () # flush
index += 1
elif isinstance(item, ResponseFunctionToolCall):
@ -1300,16 +1369,17 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
output_items, "stop", tool_search_items=tool_search_items
)
]
return [
_fold_choices_into_one(
choices=choices,
tool_calls=accumulated_tool_calls,
trailing_reasoning_content=reasoning_content,
trailing_reasoning_item=pending_reasoning_item,
tool_search_items=tool_search_items,
structured_output=structured_output,
)
]
folded: Final = _fold_choices_into_one(
choices=choices,
tool_calls=accumulated_tool_calls,
trailing_reasoning_content=reasoning_content,
trailing_reasoning_items=pending_reasoning_items,
tool_search_items=tool_search_items,
structured_output=structured_output,
)
if not replay_openai_history:
return [folded]
return [_with_replay_fields(folded, output_items, completed=completed, structured_output=structured_output)]
@staticmethod
def _build_empty_choice(
@ -1443,6 +1513,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
structured_output=is_structured_output_request(
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
),
replay_openai_history=_served_provider(
cast("Mapping[str, object]", litellm_params) # cast-ok: litellm_params is an untyped dict
)
== "openai",
completed=raw_response.status == "completed",
)
response_is_incomplete: Final = raw_response.status == "incomplete" or (
@ -1506,9 +1581,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
sync_stream: bool,
json_mode: bool | None = False,
structured_output: bool = False,
replays_openai_history: bool = False,
) -> BaseModelResponseIterator:
return OpenAiResponsesToChatCompletionStreamIterator(
streaming_response, sync_stream, json_mode, structured_output=structured_output
streaming_response,
sync_stream,
json_mode,
structured_output=structured_output,
replays_openai_history=replays_openai_history,
)
def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]:
@ -1586,7 +1666,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
if original_type == "text":
converted = with_prompt_cache_breakpoint(
self._convert_content_str_to_input_text(item.get("text", ""), role),
_prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
)
result.append(converted)
verbose_logger.debug("Chat provider: text -> %s", converted)
@ -1599,14 +1679,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
cast(ChatCompletionImageObject, item), role
),
),
_prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
)
result.append(converted)
verbose_logger.debug("Chat provider: image_url -> %s", converted)
elif original_type == "input_audio":
converted = with_prompt_cache_breakpoint(
{"type": "input_audio", "input_audio": item.get("input_audio")},
_prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
)
result.append(converted)
verbose_logger.debug("Chat provider: input_audio -> %s", converted)
@ -1622,7 +1702,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
_input_file_from_file_value(
cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked
),
_prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
)
result.append(converted)
verbose_logger.debug("Chat provider: file -> %s", converted)
@ -1646,7 +1726,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
# Default to input_text for unknown types
converted = with_prompt_cache_breakpoint(
self._convert_content_str_to_input_text(str(item.get("text", item)), role),
_prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
)
result.append(converted)
verbose_logger.debug("Chat provider: unknown(%s) -> %s", original_type, converted)
@ -1890,8 +1970,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
sync_stream: bool,
json_mode: bool | None = False,
structured_output: bool = False,
replays_openai_history: bool = False,
):
super().__init__(streaming_response, sync_stream, json_mode)
self._replays_openai_history: Final = replays_openai_history
self._chat_completion_id: str | None = None
self._served_service_tier: str | None = None
self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state
@ -1937,6 +2019,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
def translate_responses_chunk_to_openai_stream(
parsed_chunk: dict | BaseModel,
tool_call_index_map: dict[int, int] | None = None, # mutable-ok: per-stream state, remapped in place
replays_openai_history: bool = False,
structured_output: bool = False,
) -> "ModelResponseStream":
"""
Translate a Responses API streaming chunk to OpenAI chat completion streaming format.
@ -1944,6 +2028,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
Args:
parsed_chunk: Dict containing the Responses API event chunk
tool_call_index_map: Per-stream output_index -> sequential tool_call index map
replays_openai_history: Record the completed output's item sequence for the next turn
structured_output: Only the first message item became content
Returns:
ModelResponseStream: OpenAI-formatted streaming chunk
@ -2154,6 +2240,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
terminal_reasoning_items_typed: Final = _as_chat_reasoning_items(
_reasoning_items_from_output_items(output_items)
)
output_payload: Final = cast(object, output_items) # cast-ok: untyped
tool_search_fields: Final = _tool_search_provider_fields(
_tool_search_items_from_output_items(output_payload)
)
usage = None
if response_data.get("usage"):
@ -2167,10 +2257,17 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
StreamingChoices(
index=0,
delta=Delta(
content="",
content=None if replays_openai_history else "",
reasoning_items=terminal_reasoning_items_typed,
provider_specific_fields=_tool_search_provider_fields(
_tool_search_items_from_output_items(cast(object, output_items)) # cast-ok: untyped
provider_specific_fields=(
completed_stream_history(
output_payload,
tool_search_fields,
completed=event_type == "response.completed",
structured_output=structured_output,
)
if replays_openai_history
else tool_search_fields
),
),
finish_reason=finish_reason,
@ -2216,7 +2313,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
self._empty_content_chunk()
if self._is_text_delta_after_structured_answer(chunk)
else OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(
chunk, tool_call_index_map=self._tool_call_index_map
chunk,
tool_call_index_map=self._tool_call_index_map,
replays_openai_history=self._replays_openai_history,
structured_output=self._structured_output,
)
)
if self._structured_output and _is_message_item_done(chunk):

View file

@ -72,6 +72,8 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4
CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint")
OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6)
_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?")
_FINE_TUNED_BASE_PATTERN: Final = re.compile(r"^ft:([^:]+)")
_O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)")
OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset(
{"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"}
)
@ -125,6 +127,10 @@ def _served_provider_for_model(model: str) -> str | None:
def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool:
return openai_prompt_cache_breakpoint_support(model, custom_llm_provider) is True
def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str | None = None) -> bool | None:
hosted_flag: Final = (
None
if custom_llm_provider is None
@ -135,9 +141,19 @@ def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str
model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model)
if model_map_flag is not None:
return model_map_flag
version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower())
if version_match is None:
fine_tuned_base: Final = _FINE_TUNED_BASE_PATTERN.match(model.rsplit("/", 1)[-1])
if fine_tuned_base is not None:
return False if _named_model_breakpoint_support(fine_tuned_base.group(1)) is False else None
return _named_model_breakpoint_support(model)
def _named_model_breakpoint_support(model: str) -> bool | None:
bare_model: Final = model.rsplit("/", 1)[-1].lower()
if _O_SERIES_PATTERN.match(bare_model) is not None:
return False
version_match: Final = _GPT_VERSION_PATTERN.match(bare_model)
if version_match is None:
return None
version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0))
return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION

View file

@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import (
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error
from litellm.types.utils import LlmProviders
from ..exceptions import (
@ -2762,6 +2763,8 @@ def exception_type(
exception_provider=exception_provider,
extra_information=extra_information,
)
if custom_llm_provider == "bigmodel":
raise_bigmodel_business_error(model, f"{exception_provider} - {error_str}", original_exception) # pyright: ignore[reportUnknownArgumentType] # exception_type takes untyped parameters
if custom_llm_provider == "openrouter":
_map_openrouter_exception(
model=model,

View file

@ -794,6 +794,13 @@ def _get_openai_compatible_provider_info(
api_base,
dynamic_api_key,
) = litellm.XAIChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "bigmodel":
from litellm.llms.bigmodel.common_utils import get_bigmodel_api_key, get_bigmodel_url
(
api_base, # rebind-ok: existing provider-resolution pipeline resolves the API base in place
dynamic_api_key, # rebind-ok: existing provider-resolution pipeline resolves the API key in place
) = get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key)
elif custom_llm_provider == "zai":
(
api_base,

View file

@ -120,7 +120,7 @@ def is_unbilled_non_inference_call(
origin: Final = metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) if metadata is not None else None
if origin == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN:
return False
if not is_background_response(response):
if not is_background_response(_finished_response(response)):
return True
return origin == BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN or _is_managed_background_call(metadata)

View file

@ -1,8 +1,9 @@
from collections.abc import Mapping
from collections.abc import Iterable, Iterator, Mapping
from typing import Final, cast
import litellm
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
def _form_field_value(value: object) -> str:
@ -153,24 +154,33 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
if custom_llm_provider not in litellm.models_by_provider:
return []
known_models: Final = litellm.models_by_provider.get(custom_llm_provider, [])
model_costs: Final = []
known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ())
ranked_rows: Final = (
(route_rank, cost, model)
for model, cost in _chat_row_costs(known_models, custom_llm_provider)
if (route_rank := _route_rank(model, custom_llm_provider)) is not None
)
return [model for _, _, model in sorted(ranked_rows, key=lambda row: row[:2])[:n]]
for model in known_models:
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]:
for model in models:
try:
model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
except Exception:
continue
if model_info.get("mode") != "chat":
continue
_cost = (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
model_costs.append((model, _cost))
yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
# Sort by cost (ascending)
model_costs.sort(key=lambda x: x[1])
# Return the top n cheapest models
return [model for model, _ in model_costs[:n]]
def _route_rank(model: str, custom_llm_provider: str) -> int | None:
routed_provider: Final = inferred_provider(model)
if routed_provider is None:
return 1
if routed_provider == inferred_provider(f"{custom_llm_provider}/{model}"):
return 0
return None
def get_proxy_server_request_headers(litellm_params: dict | None) -> dict:

View file

@ -17,7 +17,7 @@ from pathlib import Path
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
from pydantic import TypeAdapter
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm import verbose_logger
@ -2073,6 +2073,19 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key
_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint)
def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
if marker is None or not drop_params:
return marker
try:
return _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
except ValidationError:
verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
return None
def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
"""Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block."""
if block.get("cache_control") is None:

View file

@ -1938,6 +1938,16 @@ class CustomStreamWrapper:
model_response: Final = self.model_response_creator(
hidden_params=None if self.received_model_id is None else {"received_model_id": self.received_model_id}
)
served_tier: Final = next(
(
tier
for chunk in reversed(self.chunks)
if isinstance(tier := getattr(chunk, "service_tier", None), str) and tier
),
None,
)
if served_tier is not None:
setattr(model_response, "service_tier", served_tier) # noqa: B010 # pydantic extra, not a declared field
_finish_reason: Final = self.received_finish_reason or self.intermittent_finish_reason
if _finish_reason is not None:
model_response.choices[0].finish_reason = _finish_reason

View file

@ -611,7 +611,7 @@ T_ParamSpec: Final = ParamSpec("T_ParamSpec")
T_Retval = TypeVar("T_Retval")
_COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter")
_ANTHROPIC_TOOL_KEYS_MAPPED_BY_BRIDGE: Final = frozenset(
("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming")
("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming", "custom")
)

View file

@ -121,6 +121,54 @@ _DECODED_JSON: Final = TypeAdapter(object)
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
def is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool:
if custom_llm_provider == "anthropic":
return True
model_lower: Final = model.lower()
if custom_llm_provider == "bedrock":
return "claude" in model_lower or ("arn:" in model_lower and ":bedrock:" in model_lower)
if custom_llm_provider == "azure_ai" or custom_llm_provider == "vertex_ai":
return "claude" in model_lower
return False
def is_claude_model_target(custom_llm_provider: str | None, model: str) -> bool:
return "claude" in model.lower() and is_claude_tool_target(custom_llm_provider, model)
def _hoistable_defer_loading(tool_dict: Mapping[object, object], custom: object) -> bool | None:
if not isinstance(custom, dict) or "defer_loading" in tool_dict:
return None
defer_loading: Final = OBJECT_MAPPING_ADAPTER.validate_python(custom).get("defer_loading")
return defer_loading if isinstance(defer_loading, bool) else None
def _tool_without_custom_field(tool: object, hoist_defer_loading: bool, drop_custom: bool) -> object:
if not isinstance(tool, dict):
return tool
if "custom" not in tool:
return tool
tool_dict: Final = OBJECT_MAPPING_ADAPTER.validate_python(tool)
custom: Final = tool_dict["custom"]
without_custom: Final = {key: value for key, value in tool_dict.items() if key != "custom"}
defer_loading: Final = _hoistable_defer_loading(tool_dict, custom) if hoist_defer_loading else None
if defer_loading is None:
return without_custom if drop_custom else tool_dict
unhoisted: Final = {
key: value for key, value in OBJECT_MAPPING_ADAPTER.validate_python(custom).items() if key != "defer_loading"
}
kept_custom: Final = {"custom": unhoisted} if unhoisted and not drop_custom else {}
return {**without_custom, "defer_loading": defer_loading, **kept_custom}
def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: bool, drop_custom: bool) -> list[object]:
return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools]
def is_anthropic_messages_url(url: str) -> bool:
"""Check whether a URL addresses Anthropic's Messages API."""
parsed_url: Final = urlparse(url)

View file

@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
tools: list[dict[str, JsonValue]] | None = None,
system: JsonValue = None,
optional_params: Mapping[str, JsonValue] | None = None,
drop_params: bool = False,
) -> dict[str, JsonValue]:
"""
Handle a CountTokens request using httpx.
@ -69,6 +70,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
tools=tools,
system=system,
optional_params=optional_params,
drop_params=drop_params,
)
verbose_logger.debug("Transformed request: %s", request_body)

View file

@ -11,6 +11,7 @@ from pydantic import JsonValue
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
from litellm.llms.anthropic.count_tokens.transformation import (
AnthropicCountFields,
@ -131,6 +132,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
tools=tools,
system=system,
optional_params=count_options,
drop_params=drops_unsupported_params(litellm_params.get("drop_params")),
)
if result is not None:

View file

@ -26,7 +26,11 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
split_leading_system_run,
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers
from litellm.llms.anthropic.common_utils import (
is_claude_model_target,
merge_anthropic_beta_headers,
tools_without_custom_field,
)
from litellm.llms.anthropic.wif import resolve_anthropic_base
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
@ -372,6 +376,7 @@ class AnthropicCountTokensConfig:
tools: list[dict[str, JsonValue]] | None = None,
system: JsonValue = None,
optional_params: Mapping[str, JsonValue] | None = None,
drop_params: bool = False,
) -> dict[str, JsonValue]: # mutable-ok: provider transport requires JSON dictionaries
"""
Transform request to Anthropic CountTokens format.
@ -391,7 +396,19 @@ class AnthropicCountTokensConfig:
**MappingProxyType(
{
key: value
for key, value in (("system", counted_system), ("tools", tools))
for key, value in (
("system", counted_system),
(
"tools",
None
if tools is None
else tools_without_custom_field(
tools,
hoist_defer_loading=is_claude_model_target("anthropic", model),
drop_custom=drop_params,
),
),
)
if value is not None
}
),

View file

@ -817,6 +817,7 @@ class LiteLLMAnthropicMessagesAdapter:
"name",
"input_schema",
"description",
"custom",
"cache_control",
"strict",
"type",

View file

@ -29,13 +29,16 @@ from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
from litellm.types.router import GenericLiteLLMParams
from ...common_utils import (
OBJECT_LIST_ADAPTER,
AnthropicError,
AnthropicModelInfo,
is_claude_model_target,
merge_anthropic_beta_headers,
optionally_handle_anthropic_oauth,
requires_native_compaction_beta,
strip_advisor_blocks_from_messages,
strip_encrypted_reasoning_blocks_from_anthropic_messages,
tools_without_custom_field,
)
from .mid_conversation_system import (
as_system_content_blocks,
@ -181,6 +184,16 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
"""
return False
def hoists_wrapped_defer_loading(self, model: str) -> bool:
return is_claude_model_target(self._resolved_provider, model)
def tools_for_upstream(self, tools: Sequence[object], model: str, drop_params: object) -> Sequence[object]:
return tools_without_custom_field(
tools,
hoist_defer_loading=self.hoists_wrapped_defer_loading(model),
drop_custom=drops_unsupported_params(drop_params),
)
@staticmethod
def _filter_billing_headers_from_system(system_param):
"""
@ -700,7 +713,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
# Auto-strip advisor blocks from history if advisor tool is absent.
# Prevents Anthropic 400: advisor_tool_result in history requires advisor tool.
_tools: Final = anthropic_messages_optional_request_params.get("tools") or []
tools: Final = anthropic_messages_optional_request_params.get("tools")
_tools: Final = tools or []
_has_advisor: Final = any(isinstance(t, dict) and t.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for t in _tools)
if not _has_advisor:
messages = strip_advisor_blocks_from_messages(messages)
@ -713,7 +727,14 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
model=model,
**anthropic_messages_optional_request_params,
)
return dict(anthropic_messages_request)
if not isinstance(tools, list):
return dict(anthropic_messages_request)
return {
**anthropic_messages_request,
"tools": self.tools_for_upstream(
OBJECT_LIST_ADAPTER.validate_python(tools), model, litellm_params.drop_params
),
}
def transform_anthropic_messages_response(
self,

View file

@ -15,7 +15,11 @@ from litellm.integrations.anthropic_cache_control_hook import (
supports_openai_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint
from litellm.litellm_core_utils.prompt_templates.common_utils import (
cache_control_as_prompt_cache_breakpoint,
prompt_cache_breakpoint_for_wire,
with_prompt_cache_breakpoint,
)
from litellm.responses.utils import auto_reasoning_summary
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
from litellm.types.llms.anthropic import (
@ -90,14 +94,28 @@ def _without_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str
return {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"}
def _cache_control_as_wire_safe_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str, object]:
if "prompt_cache_breakpoint" not in block:
return cache_control_as_prompt_cache_breakpoint(block)
wire_marker: Final = prompt_cache_breakpoint_for_wire(block["prompt_cache_breakpoint"], drop_params=True)
return cache_control_as_prompt_cache_breakpoint(
with_prompt_cache_breakpoint(_without_prompt_cache_breakpoint(block), wire_marker)
)
def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, object]) -> BlockRewrite | None:
"""Client cache marks become breakpoints for a deployment that reads them, and a model that takes no breakpoints
drops raw ones under drop_params."""
drops_params: Final = drops_unsupported_params(forwarded_kwargs.get("drop_params"))
if _reads_prompt_cache_breakpoint(model, forwarded_kwargs):
return cache_control_as_prompt_cache_breakpoint
return (
_cache_control_as_wire_safe_prompt_cache_breakpoint
if drops_params
else cache_control_as_prompt_cache_breakpoint
)
if _model_takes_prompt_cache_breakpoint(model, forwarded_kwargs):
return None
if drops_unsupported_params(forwarded_kwargs.get("drop_params")):
if drops_params:
return _without_prompt_cache_breakpoint
return None

View file

@ -335,7 +335,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> tuple[str, dict]:
) -> tuple[str, dict[str, str]]:
"""
Transform the get response API request into a URL and data
@ -343,7 +343,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
- GET /v1/responses/{response_id}
"""
get_url: Final = self._construct_url_for_response_id_in_path(api_base=api_base, response_id=response_id)
data: Final[dict] = {}
data: Final[dict[str, str]] = {}
verbose_logger.debug("get response url=%s", get_url)
return get_url, data

View file

@ -13,6 +13,7 @@ from pydantic import JsonValue
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.llms.anthropic.common_utils import AnthropicError
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
AzureAIAnthropicCountTokensConfig,
@ -69,6 +70,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
tools=tools,
system=system,
optional_params=optional_params,
drop_params=drops_unsupported_params((litellm_params or {}).get("drop_params")),
)
verbose_logger.debug("Transformed request: %s", request_body)

View file

@ -69,6 +69,9 @@ class BaseResponsesAPIConfig(ABC):
def supports_encrypted_agent_messages(self) -> bool:
return False
def supports_streamed_retrieval(self) -> bool:
return self.custom_llm_provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI)
def sign_request(
self,
headers: dict,
@ -209,7 +212,7 @@ class BaseResponsesAPIConfig(ABC):
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> tuple[str, dict]:
) -> tuple[str, dict[str, str]]:
pass
@abstractmethod
@ -268,6 +271,9 @@ class BaseResponsesAPIConfig(ABC):
"""Returns True if litellm should fake a stream for the given model and stream value"""
return False
def may_answer_stream_with_json(self) -> bool:
return False
def supports_native_websocket(self) -> bool:
"""
Returns True if the provider has a native WebSocket endpoint for Responses API.

View file

@ -15,7 +15,11 @@ from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias
from typing_extensions import ReadOnly, TypedDict
from typing_extensions import (
ReadOnly,
TypedDict,
TypeIs, # noqa: TID251 # narrows the caller's tools without copying them
)
if TYPE_CHECKING:
from botocore.eventstream import EventStreamMessage
@ -573,6 +577,26 @@ def strip_unsupported_bedrock_invoke_output_config_keys(
request_body["output_config"] = {"format": preserved_format} # rebind-ok: out-param
_TOOL_ITEMS_ADAPTER: Final = TypeAdapter(tuple[object, ...])
def _is_str_keyed_dict(
value: object,
) -> TypeIs[dict[str, object]]: # guard-ok: JSON tool objects # mutable-ok: fallbacks reuse tools edited in place
return isinstance(value, dict)
def _normalize_custom_field_on_tool(tool: object) -> None:
if not _is_str_keyed_dict(tool):
return
custom: Final = tool.pop("custom", None)
if not _is_str_keyed_dict(custom) or "defer_loading" in tool:
return
deferred: Final = custom.get("defer_loading")
if isinstance(deferred, bool):
tool["defer_loading"] = deferred # rebind-ok: a fallback reuses the caller's tool, as on main
def normalize_custom_field_on_tools(request_body: dict) -> None:
"""
Drop the ``custom`` field from each tool, first hoisting a boolean
@ -590,15 +614,8 @@ def normalize_custom_field_on_tools(request_body: dict) -> None:
tools: Final = request_body.get("tools")
if not tools or not isinstance(tools, list):
return
for tool in tools:
if not isinstance(tool, dict):
continue
custom: dict[str, object] | None = tool.pop("custom", None)
if not isinstance(custom, dict) or "defer_loading" in tool:
continue
deferred: object = custom.get("defer_loading")
if isinstance(deferred, bool):
tool["defer_loading"] = deferred
for tool in _TOOL_ITEMS_ADAPTER.validate_python(tools):
_normalize_custom_field_on_tool(tool)
_TOOL_DICTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])

View file

@ -8,6 +8,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import tools_without_custom_field
from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES, AnthropicCountTokensConfig
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing
from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url
@ -26,6 +27,7 @@ class MantleCountTokensRequest(TypedDict):
_COUNT_REQUEST: Final = TypeAdapter(MantleCountTokensRequest)
_COUNT_RESPONSE: Final = TypeAdapter(dict[str, JsonValue])
_COUNT_OPTIONS: Final[TypeAdapter[Mapping[str, JsonValue]]] = TypeAdapter(Mapping[str, JsonValue])
_TOOLS: Final = TypeAdapter(list[dict[str, JsonValue]])
class BedrockMantleCountTokensHandler(BaseAWSLLM):
@ -59,10 +61,15 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM):
self.get_aws_region_name, optional_params=litellm_params, model=resolved_model, model_id=None
)
endpoint_url: Final = await run_aws_signing(self.get_mantle_count_tokens_endpoint, aws_region_name)
tools: Final = request.get("tools")
body: Final = self._anthropic_config.transform_request_to_count_tokens(
model=resolved_model,
messages=request["messages"],
tools=request.get("tools"),
tools=None
if tools is None
else _TOOLS.validate_python(
tools_without_custom_field(tools, hoist_defer_loading=True, drop_custom=True)
),
system=request.get("system"),
optional_params=_COUNT_OPTIONS.validate_python(
{name: value for name, value in request_data.items() if name in COUNT_TOKEN_OPTION_NAMES}

View file

@ -12,6 +12,7 @@ from typing import Final, Literal
from pydantic import JsonValue
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import tools_without_custom_field
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import get_bedrock_base_model
@ -194,7 +195,15 @@ class BedrockCountTokensConfig(BaseAWSLLM):
# For InvokeModel, we need to provide the raw body that would be sent to the model
# Remove the 'model' field from the body as it's not part of the model input
body_data: Final = {k: v for k, v in request_data.items() if k != "model"}
tools: Final = request_data.get("tools")
body_data: Final = {
**{k: v for k, v in request_data.items() if k != "model"},
**(
{"tools": tools_without_custom_field(tools, hoist_defer_loading=True, drop_custom=True)}
if isinstance(tools, list)
else {}
),
}
if "messages" in body_data:
# Bedrock validates the body against the model's InvokeModel schema;

View file

@ -102,6 +102,9 @@ class AmazonAnthropicClaudeMessagesConfig(
def should_strip_billing_metadata(self) -> bool:
return True
def tools_for_upstream(self, tools: Sequence[object], model: str, drop_params: object) -> Sequence[object]:
return tools
def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool:
return True

View file

View file

View file

@ -0,0 +1,59 @@
from collections.abc import Mapping
from typing import Final
from litellm.llms.zai.chat.transformation import ZAIChatConfig
from ..common_utils import get_bigmodel_api_key, get_bigmodel_url, is_unsupported_disabled_thinking
_SUPPORTED_PARAMS: Final = (
"max_tokens",
"max_completion_tokens",
"stream",
"stream_options",
"temperature",
"top_p",
"stop",
"tools",
"tool_choice",
"response_format",
"thinking",
"reasoning_effort",
)
class BigModelChatConfig(ZAIChatConfig):
@property
def custom_llm_provider(self) -> str:
return "bigmodel"
def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key)
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited interface requires a list
return list(_SUPPORTED_PARAMS)
def map_openai_params(
self,
non_default_params: Mapping[str, object],
optional_params: Mapping[str, object],
model: str,
drop_params: bool,
) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body
mapped: Final = {
**optional_params,
**{
("max_tokens" if key == "max_completion_tokens" else key): value
for key, value in non_default_params.items()
if key in _SUPPORTED_PARAMS and key != "thinking"
},
}
if "thinking" not in non_default_params:
return mapped
thinking: Final = non_default_params["thinking"]
if is_unsupported_disabled_thinking(model, thinking):
return {
**mapped,
"reasoning_effort": mapped.get("reasoning_effort", "low"),
"extra_body": {"thinking": {"type": "enabled"}},
}
return {**mapped, "extra_body": {"thinking": thinking}}

View file

@ -0,0 +1,92 @@
import re
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final, Literal
from urllib.parse import urlsplit, urlunsplit
import httpx
from pydantic import JsonValue, TypeAdapter, ValidationError
import litellm
from litellm.exceptions import ContentPolicyViolationError, ContextWindowExceededError
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
BIGMODEL_API_BASE: Final = "https://open.bigmodel.cn"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_BUSINESS_CODE_IN_ERROR: Final = re.compile(r"""['"]code['"]\s*:\s*['"]?(\d{4})\b""")
_CODING_CHAT_PATH: Final = "/api/coding/paas/v4"
_API_PATHS: Final = (
"/api/coding/paas/v4/chat/completions",
_CODING_CHAT_PATH,
"/api/paas/v4/chat/completions",
"/api/anthropic/v1/messages",
"/api/anthropic/v1",
"/api/v1/responses",
"/api/paas/v4",
"/api/anthropic",
"/api/v1",
"/api",
)
_ENDPOINT_PATHS: Final = MappingProxyType(
{
"chat": "/api/paas/v4",
"responses": "/api/v1/responses",
"messages": "/api/anthropic/v1/messages",
}
)
def get_bigmodel_api_key(api_key: str | None) -> str | None:
return api_key or get_secret_str("BIGMODEL_API_KEY")
def get_bigmodel_url(api_base: str | None, endpoint: Literal["chat", "responses", "messages"]) -> str:
resolved_base: Final = api_base or get_secret_str("BIGMODEL_API_BASE") or BIGMODEL_API_BASE
parsed: Final = urlsplit(resolved_base)
path: Final = parsed.path.rstrip("/")
suffix: Final = next((candidate for candidate in _API_PATHS if path.endswith(candidate)), "")
endpoint_path: Final = (
_CODING_CHAT_PATH if endpoint == "chat" and suffix.startswith(_CODING_CHAT_PATH) else _ENDPOINT_PATHS[endpoint]
)
return urlunsplit(
(parsed.scheme, parsed.netloc, f"{path.removesuffix(suffix)}{endpoint_path}", parsed.query, parsed.fragment)
)
def bigmodel_json_body(response: httpx.Response) -> Mapping[str, JsonValue]:
try:
return _JSON_OBJECT.validate_json(response.content)
except ValidationError:
raise BaseLLMException(message=response.text, status_code=response.status_code, headers=response.headers)
def is_unsupported_disabled_thinking(model: str, thinking: object) -> bool:
return thinking == {"type": "disabled"} and litellm.utils.supports_factory(
model=model, custom_llm_provider="bigmodel", key="thinking_always_on"
)
def bigmodel_business_error(
code: JsonValue, message: str, model: str, response: httpx.Response | None
) -> ContextWindowExceededError | ContentPolicyViolationError | None:
match code:
case 1261 | "1261":
return ContextWindowExceededError(message=message, model=model, llm_provider="bigmodel", response=response)
case 1301 | "1301":
return ContentPolicyViolationError(message=message, model=model, llm_provider="bigmodel", response=response)
case _:
return None
def raise_bigmodel_business_error(model: object, error_str: str, original_exception: object) -> None:
found: Final = _BUSINESS_CODE_IN_ERROR.search(error_str)
response: Final = getattr(original_exception, "response", None)
error: Final = bigmodel_business_error(
found.group(1) if found else None,
error_str,
str(model),
response if isinstance(response, httpx.Response) else None,
)
if error is not None:
raise error

View file

@ -0,0 +1,206 @@
import json
from collections.abc import AsyncIterator, Mapping, Sequence
from typing import Final
import httpx
from openai._streaming import ServerSentEvent, SSEDecoder
from pydantic import JsonValue, TypeAdapter, ValidationError
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.openai_like.messages.transformation import (
DEFAULT_ANTHROPIC_API_VERSION,
OpenAILikeAnthropicMessagesConfig,
)
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
from litellm.types.router import GenericLiteLLMParams
from ..common_utils import (
bigmodel_json_body,
get_bigmodel_api_key,
get_bigmodel_url,
is_unsupported_disabled_thinking,
)
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_OUTPUT_CONFIG: Final = TypeAdapter(dict[str, object])
def _with_lowest_effort_thinking(
params: Mapping[str, object],
) -> dict[str, object]: # mutable-ok: the parent transform takes a request dict
requested_output_config: Final = params.get("output_config")
output_config: Final = {
**({} if "reasoning_effort" in params else {"effort": "low"}),
**(
_OUTPUT_CONFIG.validate_python(requested_output_config) if isinstance(requested_output_config, dict) else {}
),
}
return {**params, "thinking": {"type": "adaptive"}, **({"output_config": output_config} if output_config else {})}
def _normalize_message(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: # mutable-ok: native JSON response
usage: Final = body.get("usage")
if not isinstance(usage, dict):
return dict(body)
server_tools: Final = usage.get("server_tool_use")
if not isinstance(server_tools, dict) or "web_fetch_requests" in server_tools:
return dict(body)
normalized: Final[JsonValue] = {
**body,
"usage": {**usage, "server_tool_use": {"web_fetch_requests": 0, **server_tools}},
}
return normalized
class _BigModelMessagesStream(httpx.AsyncByteStream):
def __init__(self, response: httpx.Response) -> None:
self._response = response
async def __aiter__(self) -> AsyncIterator[bytes]:
async for event in SSEDecoder().aiter_bytes(self._response.aiter_bytes()):
yield _normalize_event(event)
async def aclose(self) -> None:
await self._response.aclose()
def _json_object(data: str) -> Mapping[str, JsonValue] | None:
try:
return _JSON_OBJECT.validate_json(data)
except ValidationError:
return None
def _normalized_data(data: str) -> str:
body: Final = _json_object(data)
if body is None:
return data
message: Final = body.get("message")
normalized: Final[JsonValue] = (
{**body, "message": _normalize_message(message)}
if body.get("type") == "message_start" and isinstance(message, dict)
else _normalize_message(body)
)
return data if normalized == body else json.dumps(normalized, ensure_ascii=False)
def _normalize_event(event: ServerSentEvent) -> bytes:
metadata: Final = "".join(
f"{name}: {value}\n"
for name, value in (("event", event.event), ("id", event.id), ("retry", event.retry))
if value is not None
)
data_lines: Final = "".join(f"data: {line}\n" for line in _normalized_data(event.data).split("\n"))
return f"{metadata}{data_lines}\n".encode()
class BigModelMessagesConfig(OpenAILikeAnthropicMessagesConfig):
@property
def custom_llm_provider(self) -> str:
return "bigmodel"
def is_native_messages_route(self) -> bool:
return True
def should_strip_billing_metadata(self) -> bool:
return True
def transform_anthropic_messages_request(
self,
model: str,
messages: list[dict[str, object]], # mutable-ok: matches dict-typed base signature
anthropic_messages_optional_request_params: dict[str, object], # mutable-ok: matches dict-typed base signature
litellm_params: GenericLiteLLMParams,
headers: dict[str, str], # mutable-ok: matches dict-typed base signature
) -> dict[str, object]: # mutable-ok: matches dict-typed base signature
request_params: Final = (
_with_lowest_effort_thinking(anthropic_messages_optional_request_params)
if is_unsupported_disabled_thinking(model, anthropic_messages_optional_request_params.get("thinking"))
else anthropic_messages_optional_request_params
)
return super().transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the parent transform returns a bare dict
model, messages, request_params, litellm_params, headers
)
def transform_anthropic_messages_response(
self, model: str, raw_response: httpx.Response, logging_obj: Logging
) -> AnthropicMessagesResponse:
normalized: Final = httpx.Response(
raw_response.status_code,
json=_normalize_message(bigmodel_json_body(raw_response)),
request=raw_response.request,
)
return super().transform_anthropic_messages_response(model, normalized, logging_obj)
def get_async_streaming_response_iterator(
self,
model: str,
httpx_response: httpx.Response,
request_body: dict[str, object], # mutable-ok: inherited native stream API takes a request dict
litellm_logging_obj: Logging,
) -> AsyncIterator[bytes]:
normalized: Final = httpx.Response(
httpx_response.status_code,
headers={
key: value
for key, value in httpx_response.headers.items()
if key.lower() not in ("content-encoding", "content-length")
},
stream=_BigModelMessagesStream(httpx_response),
request=httpx_response.request,
)
return super().get_async_streaming_response_iterator( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # native parent yields bytes but its iterator annotation omits the element type
model, normalized, request_body, litellm_logging_obj
)
@staticmethod
def _translate_reasoning_effort_to_anthropic(
model: str, optional_params: Mapping[str, object], max_tokens: int | None, custom_llm_provider: str
) -> None:
return None
@staticmethod
def _translate_adaptive_effort_for_non_adaptive_model(
model: str, optional_params: Mapping[str, object], max_tokens: int | None, custom_llm_provider: str
) -> None:
return None
@staticmethod
def _drop_incompatible_temperature_for_thinking(
model: str, optional_params: Mapping[str, object], custom_llm_provider: str
) -> None:
return None
def validate_anthropic_messages_environment(
self,
headers: Mapping[str, str],
model: str,
messages: Sequence[object],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> tuple[dict[str, str], str | None]: # mutable-ok: HTTP handler merges forwarded headers
resolved_key: Final = get_bigmodel_api_key(api_key)
if resolved_key is None:
raise AuthenticationError(
message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model
)
return {
**{key: value for key, value in headers.items() if key.lower() not in ("x-api-key", "authorization")},
"x-api-key": resolved_key,
"anthropic-version": DEFAULT_ANTHROPIC_API_VERSION,
"content-type": "application/json",
}, api_base
def get_complete_url(
self,
api_base: str | None,
api_key: str | None,
model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
stream: bool | None = None,
) -> str:
return get_bigmodel_url(api_base, "messages")

View file

@ -0,0 +1,263 @@
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
import httpx
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError
from typing_extensions import Never
from litellm.exceptions import AuthenticationError, UnsupportedParamsError
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.llms.openai import (
ResponseInputParam,
ResponsesAPIOptionalRequestParams,
ResponsesAPIResponse,
ResponsesAPIStreamingResponse,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
from ..common_utils import bigmodel_business_error, bigmodel_json_body, get_bigmodel_api_key, get_bigmodel_url
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_SUPPORTED_PARAMS: Final = (
"input",
"model",
"instructions",
"stream",
"temperature",
"top_p",
"max_output_tokens",
"tools",
"tool_choice",
"parallel_tool_calls",
"include",
"reasoning",
"text",
"prompt_cache_key",
"previous_response_id",
"store",
"extra_headers",
"extra_query",
"extra_body",
"timeout",
)
_STATUS_BY_BUSINESS_CODE: Final = MappingProxyType(
{
**dict.fromkeys((1000, 1001, 1003, 1005), 401),
**dict.fromkeys((1210, 1211, 1212, 1213, 1214, 1215, 1221, 1222, 1261, 1301), 400),
1220: 403,
**dict.fromkeys(
(1113, 1302, 1305, 1308, 1309, 1310, 1311, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321),
429,
),
}
)
class _RequestedTools(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
tools: JsonValue = None
tool_choice: JsonValue = None
parallel_tool_calls: JsonValue = None
def _requested_tools(logging_obj: Logging) -> _RequestedTools:
try:
return _RequestedTools.model_validate(logging_obj.model_call_details.get("optional_params") or {})
except ValidationError:
return _RequestedTools()
def _envelope_status_code(code: JsonValue) -> int:
if not isinstance(code, int) or isinstance(code, bool):
return 500
if 400 <= code <= 599:
return code
return _STATUS_BY_BUSINESS_CODE.get(code, 500)
def _normalize_output_item(item: JsonValue) -> JsonValue:
if not isinstance(item, dict) or item.get("type") not in ("message", "reasoning"):
return item
content: Final = item.get("content")
normalized: Final[JsonValue] = {"summary": [], **item} if item["type"] == "reasoning" else item
if not isinstance(content, dict):
return normalized
return {**normalized, "content": [content]}
def _normalize_response(
body: Mapping[str, JsonValue],
requested: _RequestedTools,
) -> dict[str, JsonValue]: # mutable-ok: JSON serialization boundary
usage: Final = body.get("usage")
output: Final = body.get("output")
tools: Final = body.get("tools")
tool_choice: Final = body.get("tool_choice")
parallel_tool_calls: Final = body.get("parallel_tool_calls")
input_tokens: Final = usage.get("input_tokens") if isinstance(usage, dict) else None
output_tokens: Final = usage.get("output_tokens") if isinstance(usage, dict) else None
normalized_usage: Final = (
{
"total_tokens": input_tokens + output_tokens,
**usage,
}
if isinstance(usage, dict) and isinstance(input_tokens, int) and isinstance(output_tokens, int)
else usage
)
normalized: Final[JsonValue] = {
**body,
"parallel_tool_calls": (
requested.parallel_tool_calls is not False if parallel_tool_calls is None else parallel_tool_calls
),
"tools": (requested.tools or []) if tools is None else tools,
"tool_choice": (requested.tool_choice or "auto") if tool_choice is None else tool_choice,
**({"usage": normalized_usage} if "usage" in body else {}),
**({"output": [_normalize_output_item(item) for item in output]} if isinstance(output, list) else {}),
}
return normalized
def _normalize_stream_part(part: JsonValue) -> JsonValue:
if not isinstance(part, dict):
return part
if part.get("type") == "output_text":
return {
"annotations": [],
"logprobs": None,
**part,
}
if part.get("type") == "reasoning_text":
return {"reasoning": part.get("text", ""), **part}
return part
class BigModelResponsesAPIConfig(OpenAIResponsesAPIConfig):
@property
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.BIGMODEL
def supports_native_file_search(self) -> bool:
return False
def supports_native_websocket(self) -> bool:
return False
def should_fake_stream(
self, model: str | None, stream: bool | None, custom_llm_provider: str | None = None
) -> bool:
return False
def may_answer_stream_with_json(self) -> bool:
return True
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited API requires a list
return list(_SUPPORTED_PARAMS)
def map_openai_params(
self,
response_api_optional_params: ResponsesAPIOptionalRequestParams,
model: str,
drop_params: bool,
) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body
return {key: value for key, value in response_api_optional_params.items() if key in _SUPPORTED_PARAMS}
def validate_environment(
self, headers: Mapping[str, str], model: str, litellm_params: GenericLiteLLMParams | None
) -> dict[str, str]: # mutable-ok: HTTP handler merges forwarded headers
resolved_key: Final = get_bigmodel_api_key(litellm_params.api_key if litellm_params else None)
if resolved_key is None:
raise AuthenticationError(
message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model
)
return {
**headers,
"Authorization": f"Bearer {resolved_key}",
"Content-Type": "application/json",
}
def get_complete_url(self, api_base: str | None, litellm_params: Mapping[str, object]) -> str:
return get_bigmodel_url(api_base, "responses")
def transform_response_api_response(
self, model: str, raw_response: httpx.Response, logging_obj: Logging
) -> ResponsesAPIResponse:
body: Final = bigmodel_json_body(raw_response)
if body.get("success") is False:
message: Final = str(body.get("msg") or raw_response.text)
raise bigmodel_business_error(body.get("code"), message, model, None) or BaseLLMException(
message=message,
status_code=_envelope_status_code(body.get("code")),
headers=raw_response.headers,
)
normalized: Final = _normalize_response(body, _requested_tools(logging_obj))
normalized_response: Final = httpx.Response(
status_code=raw_response.status_code,
headers={
key: value
for key, value in raw_response.headers.items()
if key.lower() not in ("content-encoding", "content-length")
},
json=normalized,
request=raw_response.request,
)
return super().transform_response_api_response(model, normalized_response, logging_obj)
def transform_get_response_api_response(
self, raw_response: httpx.Response, logging_obj: Logging
) -> ResponsesAPIResponse:
return self.transform_response_api_response("", raw_response, logging_obj)
def transform_streaming_response(
self, model: str, parsed_chunk: Mapping[str, object], logging_obj: Logging
) -> ResponsesAPIStreamingResponse:
chunk: Final = _JSON_OBJECT.validate_python(parsed_chunk)
response: Final = chunk.get("response")
item: Final = chunk.get("item")
normalized: Final[JsonValue] = {
**chunk,
**(
{"response": _normalize_response(response, _requested_tools(logging_obj))}
if isinstance(response, dict)
else {}
),
**({"item": _normalize_output_item(item)} if "item" in chunk else {}),
}
event_model: Final = self.get_event_model_class(str(normalized.get("type")))
match chunk.get("type"):
case "response.output_text.done":
return event_model.model_validate({"logprobs": [], **normalized})
case "response.content_part.done":
return event_model.model_validate({**normalized, "part": _normalize_stream_part(chunk.get("part"))})
case "error":
error: Final = chunk.get("error")
error_fields: Final = error if isinstance(error, dict) else chunk
normalized_error: Final[JsonValue] = {
**normalized,
"error": {"type": "api_error", **error_fields, "code": error_fields.get("code") or "unknown_error"},
}
return event_model.model_validate(normalized_error)
case _:
return event_model.model_validate(normalized)
def transform_cancel_response_api_request(
self, response_id: str, api_base: str, litellm_params: GenericLiteLLMParams, headers: Mapping[str, str]
) -> Never:
raise UnsupportedParamsError(message="BigModel does not support cancelling responses", status_code=400)
def transform_compact_response_api_request(
self,
model: str,
input: str | ResponseInputParam,
response_api_optional_request_params: Mapping[str, object],
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: Mapping[str, str],
) -> Never:
raise UnsupportedParamsError(message="BigModel does not support compacting responses", status_code=400)

View file

@ -764,18 +764,23 @@ class AsyncHTTPHandler:
follow_redirects: bool | None = None,
timeout: float | httpx.Timeout | None = None,
max_response_bytes: int | None = None,
stream: bool = False,
):
# Set follow_redirects to UseClientDefault if None
_follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT
query: Final = httpx.QueryParams(params).merge(HTTPHandler.extract_query_params(url))
request_headers: Final = httpx.Headers(headers)
params = params or {}
params.update(HTTPHandler.extract_query_params(url))
if stream:
return await self._send_streaming_get(
url, params=query, headers=request_headers, follow_redirects=follow_redirects, timeout=timeout
)
if max_response_bytes is not None:
return await self._get_with_response_limit(
url,
params=httpx.QueryParams(params),
headers=httpx.Headers(headers),
params=query,
headers=request_headers,
max_bytes=max_response_bytes,
follow_redirects=self.client.follow_redirects if follow_redirects is None else follow_redirects,
timeout=self.client.timeout if timeout is None else httpx.Timeout(timeout),
@ -783,13 +788,41 @@ class AsyncHTTPHandler:
response: Final = await self.client.get(
url,
params=params,
headers=headers,
params=query,
headers=request_headers,
follow_redirects=_follow_redirects,
timeout=timeout if timeout is not None else USE_CLIENT_DEFAULT,
)
return response
async def _send_streaming_get(
self,
url: str,
*,
params: httpx.QueryParams,
headers: httpx.Headers,
follow_redirects: bool | None,
timeout: float | httpx.Timeout | None,
) -> httpx.Response:
start_time: Final = time.time()
request: Final = self.client.build_request(
"GET", url, params=params, headers=headers, timeout=USE_CLIENT_DEFAULT if timeout is None else timeout
)
try:
response: Final = await self.client.send(
request,
stream=True,
follow_redirects=USE_CLIENT_DEFAULT if follow_redirects is None else follow_redirects,
)
_anchor_handler_to(response, self)
response.raise_for_status()
except httpx.TimeoutException as e:
_raise_timeout(e, start_time)
except httpx.HTTPStatusError as e:
await _raise_masked_async_error(e, True)
else:
return response
async def _get_with_response_limit(
self,
url: str,
@ -1468,15 +1501,20 @@ class HTTPHandler:
headers: Mapping[str, Any] | None = None,
follow_redirects: bool | None = None,
timeout: float | httpx.Timeout | None = None,
stream: bool = False,
):
# Set follow_redirects to UseClientDefault if None
_follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT
params = params or {}
params.update(self.extract_query_params(url))
query: Final = httpx.QueryParams(params).merge(self.extract_query_params(url))
if stream:
return self._send_streaming_get(
url, params=query, headers=headers, follow_redirects=follow_redirects, timeout=timeout
)
response: Final = self.client.get(
url,
params=params,
params=query,
headers=headers,
follow_redirects=_follow_redirects,
timeout=timeout if timeout is not None else USE_CLIENT_DEFAULT,
@ -1484,6 +1522,34 @@ class HTTPHandler:
return response
def _send_streaming_get(
self,
url: str,
*,
params: httpx.QueryParams,
headers: Mapping[str, str] | None,
follow_redirects: bool | None,
timeout: float | httpx.Timeout | None,
) -> httpx.Response:
start_time: Final = time.time()
request: Final = self.client.build_request(
"GET", url, params=params, headers=headers, timeout=USE_CLIENT_DEFAULT if timeout is None else timeout
)
try:
response: Final = self.client.send(
request,
stream=True,
follow_redirects=USE_CLIENT_DEFAULT if follow_redirects is None else follow_redirects,
)
_anchor_handler_to(response, self)
response.raise_for_status()
except httpx.TimeoutException as e:
_raise_timeout(e, start_time)
except httpx.HTTPStatusError as e:
_raise_masked_sync_error(e, True)
else:
return response
@staticmethod
def extract_query_params(url: str) -> dict[str, str]:
"""

View file

@ -367,12 +367,37 @@ class _SignedBodyKwargs(TypedDict, total=False):
json: ReadOnly[dict[str, object]]
def _is_sized_json_body(response: httpx.Response) -> bool:
headers: Final = httpx.Headers(response.headers)
media_types: Final = (value.split(";")[0].strip().lower() for value in headers.get_list("content-type"))
return "content-length" in headers and "application/json" in media_types
def _is_json_document(content: bytes) -> bool:
try:
return isinstance(json.loads(content), dict)
except ValueError:
return False
def _signed_body_kwargs(*, signed_body: bytes | None, data: dict[str, object]) -> _SignedBodyKwargs:
if signed_body is not None:
return {"data": signed_body}
return {"json": data}
def _answered_json(response: httpx.Response) -> bool:
return "application/json" in response.headers.get("content-type", "").lower()
def _stream_query_params(stream: bool, starting_after: int | None) -> Mapping[str, object]:
if not stream:
return {}
if starting_after is None:
return {"stream": "true"}
return {"stream": "true", "starting_after": starting_after}
def _google_genai_streaming_hidden_params(
*,
api_base: str,
@ -3025,6 +3050,17 @@ class BaseLLMHTTPHandler:
},
)
replay_as_stream: Final = partial(
MockResponsesAPIStreamingIterator,
model=model,
logging_obj=logging_obj,
responses_api_provider_config=responses_api_provider_config,
litellm_metadata=litellm_metadata,
custom_llm_provider=custom_llm_provider,
request_data=request_context,
call_type=CallTypes.responses.value,
)
try:
if is_stream_request:
response = sync_httpx_client.post(
@ -3036,7 +3072,13 @@ class BaseLLMHTTPHandler:
)
if caller_requested_stream:
if fake_stream is True:
return MockResponsesAPIStreamingIterator(
return replay_as_stream(response=response)
if not (
responses_api_provider_config.may_answer_stream_with_json()
and _is_sized_json_body(response)
and _is_json_document(response.read())
):
return SyncResponsesAPIStreamingIterator(
response=response,
model=model,
logging_obj=logging_obj,
@ -3046,17 +3088,6 @@ class BaseLLMHTTPHandler:
request_data=request_context,
call_type=CallTypes.responses.value,
)
return SyncResponsesAPIStreamingIterator(
response=response,
model=model,
logging_obj=logging_obj,
responses_api_provider_config=responses_api_provider_config,
litellm_metadata=litellm_metadata,
custom_llm_provider=custom_llm_provider,
request_data=request_context,
call_type=CallTypes.responses.value,
)
response.read()
else:
response = sync_httpx_client.post(
@ -3071,6 +3102,9 @@ class BaseLLMHTTPHandler:
provider_config=responses_api_provider_config,
)
if is_stream_request and caller_requested_stream:
return replay_as_stream(response=response)
initial_response: Final = responses_api_provider_config.transform_response_api_response(
model=model,
raw_response=response,
@ -3226,6 +3260,17 @@ class BaseLLMHTTPHandler:
},
)
replay_as_stream: Final = partial(
MockResponsesAPIStreamingIterator,
model=model,
logging_obj=logging_obj,
responses_api_provider_config=responses_api_provider_config,
litellm_metadata=litellm_metadata,
custom_llm_provider=custom_llm_provider,
request_data=request_context,
call_type=CallTypes.responses.value,
)
try:
if is_stream_request:
response = await async_httpx_client.post(
@ -3239,7 +3284,13 @@ class BaseLLMHTTPHandler:
if caller_requested_stream:
if fake_stream is True:
return MockResponsesAPIStreamingIterator(
return replay_as_stream(response=response)
if not (
responses_api_provider_config.may_answer_stream_with_json()
and _is_sized_json_body(response)
and _is_json_document(await response.aread())
):
return ResponsesAPIStreamingIterator(
response=response,
model=model,
logging_obj=logging_obj,
@ -3249,17 +3300,6 @@ class BaseLLMHTTPHandler:
request_data=request_context,
call_type=CallTypes.responses.value,
)
return ResponsesAPIStreamingIterator(
response=response,
model=model,
logging_obj=logging_obj,
responses_api_provider_config=responses_api_provider_config,
litellm_metadata=litellm_metadata,
custom_llm_provider=custom_llm_provider,
request_data=request_context,
call_type=CallTypes.responses.value,
)
await response.aread()
else:
response = await async_httpx_client.post(
@ -3276,6 +3316,9 @@ class BaseLLMHTTPHandler:
provider_config=responses_api_provider_config,
)
if is_stream_request and caller_requested_stream:
return replay_as_stream(response=response)
initial_response: Final = responses_api_provider_config.transform_response_api_response(
model=model,
raw_response=response,
@ -3493,12 +3536,20 @@ class BaseLLMHTTPHandler:
logging_obj: LiteLLMLoggingObj,
custom_llm_provider: str | None = None,
extra_headers: dict[str, object] | None = None,
extra_query: Mapping[str, object] | None = None,
extra_body: dict[str, object] | None = None,
stream: bool = False,
starting_after: int | None = None,
litellm_metadata: Mapping[str, object] = MappingProxyType({}),
timeout: float | httpx.Timeout | None = None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
_is_async: bool = False,
shared_session: Optional["ClientSession"] = None,
) -> ResponsesAPIResponse | Coroutine[object, object, ResponsesAPIResponse]:
) -> (
ResponsesAPIResponse
| SyncResponsesAPIStreamingIterator
| Coroutine[object, object, ResponsesAPIResponse | ResponsesAPIStreamingIterator]
):
"""
Get a response by ID
Uses GET /v1/responses/{response_id} endpoint in the responses API
@ -3511,7 +3562,11 @@ class BaseLLMHTTPHandler:
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
extra_query=extra_query,
extra_body=extra_body,
stream=stream,
starting_after=starting_after,
litellm_metadata=litellm_metadata,
timeout=timeout,
client=client,
shared_session=shared_session,
@ -3540,20 +3595,23 @@ class BaseLLMHTTPHandler:
litellm_params=litellm_params,
headers=headers,
)
query: Final = {**data, **(extra_query or {}), **_stream_query_params(stream, starting_after)}
## LOGGING
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"complete_input_dict": data,
"complete_input_dict": query,
"api_base": api_base,
"headers": headers,
},
)
try:
response: Final = sync_httpx_client.get(url=url, headers=headers, params=data)
response: Final = sync_httpx_client.get(
url=url, headers=headers, params=query, stream=stream, timeout=timeout if stream else None
)
response.raise_for_status()
except Exception as e:
raise self._handle_error(
@ -3561,6 +3619,20 @@ class BaseLLMHTTPHandler:
provider_config=responses_api_provider_config,
)
if stream and not _answered_json(response):
return SyncResponsesAPIStreamingIterator(
response=response,
model="",
logging_obj=logging_obj,
responses_api_provider_config=responses_api_provider_config,
litellm_metadata=dict(litellm_metadata),
custom_llm_provider=custom_llm_provider,
request_data={"litellm_params": dict(litellm_params)},
replays_stored_stream=True,
)
if stream:
response.read()
return responses_api_provider_config.transform_get_response_api_response(
raw_response=response,
logging_obj=logging_obj,
@ -3574,11 +3646,15 @@ class BaseLLMHTTPHandler:
logging_obj: LiteLLMLoggingObj,
custom_llm_provider: str | None = None,
extra_headers: dict[str, object] | None = None,
extra_query: Mapping[str, object] | None = None,
extra_body: dict[str, object] | None = None,
stream: bool = False,
starting_after: int | None = None,
litellm_metadata: Mapping[str, object] = MappingProxyType({}),
timeout: float | httpx.Timeout | None = None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
shared_session: Optional["ClientSession"] = None,
) -> ResponsesAPIResponse:
) -> ResponsesAPIResponse | ResponsesAPIStreamingIterator:
"""
Async version of get_responses
"""
@ -3616,20 +3692,23 @@ class BaseLLMHTTPHandler:
litellm_params=litellm_params,
headers=headers,
)
query: Final = {**data, **(extra_query or {}), **_stream_query_params(stream, starting_after)}
## LOGGING
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"complete_input_dict": data,
"complete_input_dict": query,
"api_base": api_base,
"headers": headers,
},
)
try:
response: Final = await async_httpx_client.get(url=url, headers=headers, params=data)
response: Final = await async_httpx_client.get(
url=url, headers=headers, params=query, stream=stream, timeout=timeout if stream else None
)
response.raise_for_status()
except Exception as e:
verbose_logger.debug("Error retrieving response: %s", e)
@ -3638,6 +3717,20 @@ class BaseLLMHTTPHandler:
provider_config=responses_api_provider_config,
)
if stream and not _answered_json(response):
return ResponsesAPIStreamingIterator(
response=response,
model="",
logging_obj=logging_obj,
responses_api_provider_config=responses_api_provider_config,
litellm_metadata=dict(litellm_metadata),
custom_llm_provider=custom_llm_provider,
request_data={"litellm_params": dict(litellm_params)},
replays_stored_stream=True,
)
if stream:
await response.aread()
return responses_api_provider_config.transform_get_response_api_response(
raw_response=response,
logging_obj=logging_obj,

View file

@ -241,7 +241,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> tuple[str, dict]:
) -> tuple[str, dict[str, str]]:
"""
Transform the get response API request into a URL and data.
@ -252,7 +252,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
"""
encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id")
url: Final = f"{api_base}/{encoded_response_id}"
data: Final[dict] = {}
data: Final[dict[str, str]] = {}
return url, data
def transform_get_response_api_response(

View file

@ -4,11 +4,17 @@ This file contains the calling OpenAI's `/v1/realtime` endpoint.
This requires websockets, and is currently only supported on LiteLLM Proxy.
"""
import asyncio
import ssl
from typing import TYPE_CHECKING, Final, cast
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final, Literal, cast
from pydantic import TypeAdapter
from litellm._logging import redact_string, verbose_logger
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.exceptions import AuthenticationError
from litellm.responses.streaming_iterator import map_stream_error_to_exception
from litellm.types.realtime import RealtimeQueryParams
from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
@ -25,6 +31,54 @@ from ..openai import OpenAIChatCompletion
if TYPE_CHECKING:
from fastapi import WebSocket
from openai.types.realtime import RealtimeError
from websockets.asyncio.client import ClientConnection
_SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object])
_CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL: Final = "invalid_model"
_CREDENTIALS_REJECTED: Final = "invalid_api_key"
def first_event_error(first_event: str | bytes) -> "RealtimeError | None":
event: Final = _SERVER_EVENT_FIELDS.validate_json(first_event)
if event.get("type") != "error":
return None
from openai.types.realtime import RealtimeErrorEvent
return RealtimeErrorEvent.model_validate(event).error
def _confirm_first_event(
first_event: str | bytes, model: str, sent_credentials: bool, model_listed_outside_realtime: bool
) -> Literal[True]:
error: Final = first_event_error(first_event)
if error is None or (
error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and not model_listed_outside_realtime
):
return True
if error.code == _CREDENTIALS_REJECTED or not sent_credentials:
raise AuthenticationError(message=error.message, llm_provider="openai", model=model)
raise map_stream_error_to_exception(error, model=model, custom_llm_provider="openai")
async def confirm_session_started(
connection: "ClientConnection", model: str, sent_credentials: bool, model_listed_outside_realtime: bool
) -> Literal[True]:
return _confirm_first_event(await connection.recv(), model, sent_credentials, model_listed_outside_realtime)
async def confirm_gateway_session_started(
connection: "ClientConnection",
model: str,
sent_credentials: bool,
model_listed_outside_realtime: bool,
first_event_wait_seconds: float,
) -> Literal[True]:
try:
first_event: Final = await asyncio.wait_for(connection.recv(), first_event_wait_seconds)
except asyncio.TimeoutError:
return True
return _confirm_first_event(first_event, model, sent_credentials, model_listed_outside_realtime)
class OpenAIRealtime(OpenAIChatCompletion):

View file

@ -1,4 +1,4 @@
from collections.abc import Callable, Collection, Mapping, Sequence
from collections.abc import Callable, Collection, Iterable, Mapping, Sequence
from functools import lru_cache
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Protocol, cast, get_type_hints
@ -10,7 +10,11 @@ from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, targets_openai_api
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
openai_prompt_cache_breakpoint_support,
targets_openai_api,
)
from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
@ -20,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
cache_control_as_prompt_cache_breakpoint,
drop_non_python_regex_patterns,
flatten_combinators_and_drop_non_python_regex_patterns,
prompt_cache_breakpoint_for_wire,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
@ -66,6 +71,7 @@ _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS: Final = frozenset({LlmProviders.AZURE,
_PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING: Final = _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS
_ROLES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"system", "developer", "user"})
_PART_TYPES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"input_text", "input_image", "input_file"})
_INPUT_PART_LIST_KEYS: Final = frozenset({"content", "output"})
_COMPACT_TRACKING_PARAMS: Final = frozenset({"metadata", "user"})
_COMPACT_REJECTED_PARAMS: Final = frozenset(
{
@ -93,6 +99,13 @@ _INPUT_ITEM: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str,
_CONTENT_PARTS: Final[TypeAdapter[Sequence[Mapping[str, object]]]] = TypeAdapter(Sequence[Mapping[str, object]])
class _DeclaredModel(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
base_model: object = None
model_info: object = None
class _DeploymentHost(BaseModel):
"""`responses()` builds the params from raw kwargs, where callers may spell the host `base_url`."""
@ -216,6 +229,63 @@ def _with_breakpoint_on_last_reading_part(
return (*parts[:last], marked_last, *parts[last + 1 :])
def _json_object_or_none(value: object) -> Mapping[str, object] | None:
return value if isinstance(value, Mapping) else None
def _json_array_or_empty(value: object) -> Sequence[object]:
return value if isinstance(value, (list, tuple)) else ()
def _item_with_wire_safe_prompt_cache_breakpoints(item: object, keeps_markers: bool) -> object:
fields: Final = _json_object_or_none(item)
if fields is None or not _carries_prompt_cache_breakpoint(fields):
return item
wire_safe_fields: Final = {
key: _parts_with_wire_safe_prompt_cache_breakpoints(key, value, keeps_markers)
for key, value in fields.items()
if key != "prompt_cache_breakpoint"
}
unchanged: Final = "prompt_cache_breakpoint" not in fields and _same_objects(
wire_safe_fields.values(), fields.values()
)
return item if unchanged else wire_safe_fields
def _same_objects(rebuilt: Iterable[object], originals: Iterable[object]) -> bool:
return all(new is old for new, old in zip(rebuilt, originals, strict=True))
def _carries_prompt_cache_breakpoint(fields: Mapping[str, object]) -> bool:
return "prompt_cache_breakpoint" in fields or any(
_parts_carry_prompt_cache_breakpoint(fields.get(key)) for key in _INPUT_PART_LIST_KEYS
)
def _parts_carry_prompt_cache_breakpoint(parts: object) -> bool:
return any(isinstance(part, Mapping) and "prompt_cache_breakpoint" in part for part in _json_array_or_empty(parts))
def _parts_with_wire_safe_prompt_cache_breakpoints(key: str, value: object, keeps_markers: bool) -> object:
if key not in _INPUT_PART_LIST_KEYS or not _parts_carry_prompt_cache_breakpoint(value):
return value
parts: Final = _json_array_or_empty(value)
wire_safe_parts: Final = tuple(_part_with_wire_safe_prompt_cache_breakpoint(part, keeps_markers) for part in parts)
return value if _same_objects(wire_safe_parts, parts) else list(wire_safe_parts)
def _part_with_wire_safe_prompt_cache_breakpoint(part: object, keeps_markers: bool) -> object:
fields: Final = _json_object_or_none(part)
if fields is None or "prompt_cache_breakpoint" not in fields:
return part
marker: Final = fields["prompt_cache_breakpoint"]
wire_marker: Final = prompt_cache_breakpoint_for_wire(marker, drop_params=True) if keeps_markers else None
if keeps_markers and wire_marker == marker:
return part
unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"}
return with_prompt_cache_breakpoint(unmarked, wire_marker)
class _ReasoningSupportEntry(LiteLLMBaseModel):
litellm_provider: str | None = None
supports_reasoning: bool | None = None
@ -544,12 +614,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
validated_input: Final = self._validate_input_param(
self._drop_bridge_minted_reasoning_items(input), drops_set_created_by=drops_params(litellm_params)
)
marked_input: Final = (
self._input_with_prompt_cache_breakpoints(validated_input)
if self._reads_prompt_cache_breakpoints(
model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options
)
else validated_input
reads_breakpoints: Final = self._reads_prompt_cache_breakpoints(
model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options
)
marker_safe_input: Final = self._input_with_wire_safe_prompt_cache_breakpoints(
model=model, input=validated_input, litellm_params=litellm_params, reads_breakpoints=reads_breakpoints
)
maps_anthropic_tool_search: Final = self._targets_openai_api(litellm_params) and any(
is_anthropic_tool_search_chat_tool(tool) for tool in tools or ()
@ -559,20 +628,71 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
if maps_anthropic_tool_search
else tools
)
replayable_input: Final = (
input_without_unanswered_anthropic_tool_search_calls(marked_input)
if maps_anthropic_tool_search
else marked_input
)
stripped_input, stripped_tools = self.remove_cache_control_flag_from_input_and_tools(
model=model, input=replayable_input, tools=mapped_tools
model=model,
input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search),
tools=mapped_tools,
)
if marker_safe_input is not validated_input:
self._strip_caller_input_as_if_unmarked(
model=model,
caller_input=validated_input,
reads_breakpoints=reads_breakpoints,
maps_anthropic_tool_search=maps_anthropic_tool_search,
)
object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools)
sanitized_tools: Final = self._sanitized_tool_schemas_for_openai(
model=model, tools=object_schema_tools, litellm_params=litellm_params
)
return self._drop_foreign_tool_call_item_ids(stripped_input), sanitized_tools
def _replayable_input(
self, input: str | ResponseInputParam, reads_breakpoints: bool, maps_anthropic_tool_search: bool
) -> str | ResponseInputParam:
marked_input: Final = self._input_with_prompt_cache_breakpoints(input) if reads_breakpoints else input
if not maps_anthropic_tool_search:
return marked_input
return input_without_unanswered_anthropic_tool_search_calls(marked_input)
def _strip_caller_input_as_if_unmarked(
self,
model: str,
caller_input: str | ResponseInputParam,
reads_breakpoints: bool,
maps_anthropic_tool_search: bool,
) -> None:
self.remove_cache_control_flag_from_input_and_tools(
model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search)
)
def _input_with_wire_safe_prompt_cache_breakpoints(
self,
model: str,
input: str | ResponseInputParam,
litellm_params: GenericLiteLLMParams,
reads_breakpoints: bool,
) -> str | ResponseInputParam:
if not isinstance(input, list) or not drops_params(litellm_params):
return input
keeps_markers: Final = reads_breakpoints or self._keeps_prompt_cache_breakpoints(
model=model, litellm_params=litellm_params
)
wire_safe_items: Final = tuple(
_item_with_wire_safe_prompt_cache_breakpoints(item, keeps_markers) for item in input
)
if _same_objects(wire_safe_items, input):
return input
wire_safe_input: Final = list(wire_safe_items)
return cast("ResponseInputParam", wire_safe_input) # cast-ok: items keep their shape, minus a rejected marker
def _keeps_prompt_cache_breakpoints(self, model: str, litellm_params: GenericLiteLLMParams) -> bool:
provider: Final = _provider_name(self.custom_llm_provider)
base_model: Final = self._declared_base_model(model=model, litellm_params=litellm_params)
known_support: Final = (
openai_prompt_cache_breakpoint_support(name, provider) for name in dict.fromkeys((base_model, model))
)
return next((support for support in known_support if support is not None), None) is not False
def _targets_openai_api(self, litellm_params: GenericLiteLLMParams) -> bool:
host: Final = _DeploymentHost.model_validate(litellm_params)
return self.custom_llm_provider == LlmProviders.OPENAI and targets_openai_api(host.api_base or host.base_url)
@ -688,14 +808,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
@staticmethod
def _without_foreign_tool_call_item_id(item: object) -> object:
if not isinstance(item, dict):
fields: Final = _json_object_or_none(item)
if fields is None:
return item
item_type: Final = item.get("type")
item_id: Final = item.get("id")
item_type: Final = fields.get("type")
item_id: Final = fields.get("id")
genuine_prefix: Final = TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE.get(item_type) if isinstance(item_type, str) else None
if genuine_prefix is None or not isinstance(item_id, str) or item_id.startswith(genuine_prefix):
return item
return {key: value for key, value in item.items() if key != "id"}
return {key: value for key, value in fields.items() if key != "id"}
def _sanitized_tool_schemas_for_openai(
self,
@ -739,6 +860,13 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
base_model: Final[object] = model_info.get("base_model") if isinstance(model_info, dict) else None
return base_model if isinstance(base_model, str) and base_model else model
@staticmethod
def _declared_base_model(model: str, litellm_params: GenericLiteLLMParams) -> str:
declared: Final = _DeclaredModel.model_validate(litellm_params)
model_info: Final = _json_object_or_none(declared.model_info)
candidates: Final = (declared.base_model, None if model_info is None else model_info.get("base_model"))
return next((name for name in candidates if isinstance(name, str) and name), model)
@staticmethod
def _sanitized_tool_entry(
entry: Mapping[str, object],
@ -1094,7 +1222,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> tuple[str, dict]:
) -> tuple[str, dict[str, str]]:
"""
Transform the get response API request into a URL and data
@ -1103,7 +1231,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
"""
encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id")
url: Final = f"{api_base}/{encoded_response_id}"
data: Final[dict] = {}
data: Final[dict[str, str]] = {}
return url, data
def transform_get_response_api_response(

View file

@ -37,6 +37,9 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig):
def is_native_messages_route(self) -> bool:
return False
def hoists_wrapped_defer_loading(self, model: str) -> bool:
return False
def validate_anthropic_messages_environment(
self,
headers: dict[str, str],

View file

@ -6,9 +6,12 @@ Unlike Gemini models which use Google's token counting API, partner models use
their respective publisher-specific count-tokens endpoints.
"""
from collections.abc import Mapping
from typing import Any, Final
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import OBJECT_LIST_ADAPTER, is_claude_model_target, tools_without_custom_field
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
@ -20,6 +23,22 @@ _COUNT_TOKENS_PUBLISHERS: Final = (
)
def count_tokens_request_body(
request_data: Mapping[str, object], model: str, drop_params: object
) -> Mapping[str, object]:
tools: Final = request_data.get("tools")
if not isinstance(tools, list):
return request_data
return {
**request_data,
"tools": tools_without_custom_field(
OBJECT_LIST_ADAPTER.validate_python(tools),
hoist_defer_loading=is_claude_model_target("vertex_ai", model),
drop_custom=drops_unsupported_params(drop_params),
),
}
def count_tokens_publisher(model: str) -> str | None:
return next((publisher for marker, publisher in _COUNT_TOKENS_PUBLISHERS if marker in model), None)
@ -169,7 +188,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
response: Final = await async_client.post(
endpoint_url,
headers=headers,
json=request_data,
json=dict(count_tokens_request_body(request_data, model, litellm_params.get("drop_params"))),
timeout=30.0,
)

View file

@ -313,10 +313,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> tuple[str, dict]:
) -> tuple[str, dict[str, str]]:
encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id")
url: Final = f"{api_base}/{encoded_response_id}"
data: Final[dict] = {}
data: Final[dict[str, str]] = {}
return url, data
def transform_get_response_api_response(

View file

@ -234,6 +234,7 @@ from .litellm_core_utils.streaming_chunk_builder_utils import (
ChunkProcessor as ChunkProcessor, # noqa: PLC0414 # public re-export
)
from .llms.anthropic.chat import AnthropicChatCompletion
from .llms.anthropic.common_utils import is_claude_tool_target
from .llms.azure.audio_transcriptions import AzureAudioTranscription
from .llms.azure.azure import AzureChatCompletion, check_dynamic_azure_params
from .llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion
@ -1135,17 +1136,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o
_ANTHROPIC_ONLY_TOOL_KEYS: Final = frozenset({"input_examples", "eager_input_streaming"})
def _is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool:
if custom_llm_provider == "anthropic":
return True
model_lower: Final = model.lower()
if custom_llm_provider == "bedrock":
return "claude" in model_lower or ("arn:" in model_lower and ":bedrock:" in model_lower)
if custom_llm_provider == "azure_ai" or custom_llm_provider == "vertex_ai":
return "claude" in model_lower
return False
def _history_without_unanswered_tool_search_calls(
messages: Sequence[object], custom_llm_provider: str | None, provider_config: BaseConfig
) -> tuple[object, ...] | None:
@ -2672,10 +2662,13 @@ def _complete_custom_openai(
# for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
api_base = _resolve_openai_api_base(api_base)
if custom_llm_provider == "bigmodel" and not api_key:
raise litellm.AuthenticationError(
message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model
)
organization = (
organization
or litellm.organization
or get_secret("OPENAI_ORGANIZATION")
or (None if custom_llm_provider == "bigmodel" else litellm.organization or get_secret("OPENAI_ORGANIZATION"))
or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
)
openai.organization = organization
@ -5749,7 +5742,7 @@ def completion(
}
)
if not _is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model):
if not is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model):
tools = _drop_anthropic_only_tool_keys(tools=tools)
if provider_specific_header is not None:
@ -6299,6 +6292,7 @@ def completion(
or custom_llm_provider == "nebius"
or custom_llm_provider == "wandb"
or custom_llm_provider == "clarifai"
or custom_llm_provider == "bigmodel"
or custom_llm_provider in litellm.openai_compatible_providers
or JSONProviderRegistry.exists(custom_llm_provider) # JSON-configured providers
or "ft:gpt-3.5-turbo" in model # finetune gpt-3.5-turbo

View file

@ -111,6 +111,7 @@ _FEATURES: Final[detectors.Detectors] = MappingProxyType(
Feature.CACHE_CONTROL_INJECTION: detectors.cache_control_injection,
Feature.STREAM_COST_INJECTION: route_detectors.stream_cost_injection,
Feature.THINKING_SIGNATURE_RETRY: route_detectors.thinking_signature_retry,
Feature.TOOL_CUSTOM_FIELD: route_detectors.tool_custom_field,
}
)

View file

@ -78995,5 +78995,81 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"bigmodel/glm-5.3": {
"input_cost_per_token": 1.1428571428571428e-06,
"output_cost_per_token": 4e-06,
"cache_read_input_token_cost": 2.857142857142857e-07,
"litellm_provider": "bigmodel",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"mode": "chat",
"source": "https://docs.bigmodel.cn/cn/guide/start/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"thinking_always_on": true,
"metadata": {
"comment": "Official API list price checked 2026-10-27: input 8, output 28, cached input 2 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded."
},
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses",
"/v1/messages"
],
"supports_native_streaming": true
},
"bigmodel/glm-5.3-flash": {
"input_cost_per_token": 1.142857142857143e-07,
"output_cost_per_token": 4e-07,
"cache_read_input_token_cost": 3.2857142857142855e-08,
"litellm_provider": "bigmodel",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"mode": "chat",
"source": "https://docs.bigmodel.cn/cn/guide/start/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"thinking_always_on": true,
"supports_vision": true,
"supports_video_input": true,
"metadata": {
"comment": "Official API list price checked 2026-10-27: input 0.8, output 2.8, cached input 0.23 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded."
},
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses",
"/v1/messages"
],
"supports_native_streaming": true
},
"bigmodel/glm-5.3-flashx": {
"input_cost_per_token": 2.857142857142857e-07,
"output_cost_per_token": 1e-06,
"cache_read_input_token_cost": 8.142857142857142e-08,
"litellm_provider": "bigmodel",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"mode": "chat",
"source": "https://docs.bigmodel.cn/cn/guide/start/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"thinking_always_on": true,
"supports_vision": true,
"supports_video_input": true,
"metadata": {
"comment": "Official API list price checked 2026-10-27: input 2, output 7, cached input 0.57 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded."
},
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses",
"/v1/messages"
],
"supports_native_streaming": true
}
}

View file

@ -5326,6 +5326,32 @@
"a2a": false,
"generateContent": false
}
},
"bigmodel": {
"display_name": "BigModel (Zhipu AI) (`bigmodel`)",
"url": "https://docs.bigmodel.cn/cn/guide/develop/responses/introduction",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
}
},
"endpoints": {

View file

@ -33639,7 +33639,7 @@
]
},
"get": {
"description": "Get a response by ID.\n\nSupports both:\n- Polling IDs (litellm_poll_*): Returns cumulative cached content from background responses\n- Provider response IDs: Passes through to provider API\n\nFollows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/get\n\n```bash\n# Get polling response\ncurl -X GET http://localhost:4000/v1/responses/litellm_poll_abc123 -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n\n# Get provider response\ncurl -X GET http://localhost:4000/v1/responses/resp_abc123 -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n```",
"description": "Get a response by ID.\n\nSupports both:\n- Polling IDs (litellm_poll_*): Returns cumulative cached content from background responses\n- Provider response IDs: Passes through to provider API\n\nFollows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/get\n\n```bash\n# Get polling response\ncurl -X GET http://localhost:4000/v1/responses/litellm_poll_abc123 -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n\n# Get provider response\ncurl -X GET http://localhost:4000/v1/responses/resp_abc123 -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n\n# Resume a background response's event stream after sequence number 7\ncurl -N \"http://localhost:4000/v1/responses/resp_abc123?stream=true&starting_after=7\" -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n```",
"operationId": "get_response_openai_v1_responses__response_id__get",
"parameters": [
{
@ -33650,6 +33650,46 @@
"title": "Response Id",
"type": "string"
}
},
{
"description": "Replay the response's events as server-sent events (OpenAI, Azure, and Azure AI Foundry)",
"in": "query",
"name": "stream",
"required": false,
"schema": {
"type": "boolean"
}
},
{
"description": "With stream=true, replay only the events after this sequence number",
"in": "query",
"name": "starting_after",
"required": false,
"schema": {
"minimum": 0,
"type": "integer"
}
},
{
"description": "Additional output data to include, forwarded to the provider",
"in": "query",
"name": "include",
"required": false,
"schema": {
"items": {
"type": "string"
},
"type": "array"
}
},
{
"description": "With stream=true, whether delta events carry obfuscation padding",
"in": "query",
"name": "include_obfuscation",
"required": false,
"schema": {
"type": "boolean"
}
}
],
"responses": {
@ -33659,6 +33699,12 @@
"schema": {
"$ref": "#/components/schemas/ResponsesAPIResponse"
}
},
"text/event-stream": {
"schema": {
"description": "Server sent events when stream=true",
"type": "string"
}
}
},
"description": "Successful Response"

View file

@ -20,6 +20,8 @@ from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager, get_valid_models
LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders)
_CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields)
@ -458,23 +460,21 @@ def get_known_models_from_wildcard(
# add model prefix to wildcard models
wildcard_models = [f"{model_prefix}{model.removeprefix(provider_id_prefix)}" for model in wildcard_models]
known_providers: Final = {provider.value for provider in LlmProviders}
suffix_appended_wildcard_models: Final = []
for model in wildcard_models:
if not model.startswith(f"{wildcard_provider_prefix}/"):
# `get_provider_models` returns provider-prefixed ids (e.g. "ollama/gemma3:1b").
# When the wildcard uses a custom prefix (e.g. "ollama_server1/*" to distinguish
# multiple instances), replace that existing provider prefix instead of stacking
# both, which would otherwise yield an uncallable "ollama_server1/ollama/gemma3:1b".
# Only strip the leading segment when it is a known provider, so ids whose first
# segment is an org rather than a provider (e.g. "meta-llama/Llama-3-8B") keep it.
leading, sep, model_suffix = model.partition("/")
if sep and leading in known_providers:
model = f"{wildcard_provider_prefix}/{model_suffix}"
else:
model = f"{wildcard_provider_prefix}/{model}"
suffix_appended_wildcard_models.append(model)
return list(dict.fromkeys(suffix_appended_wildcard_models))
return list(
dict.fromkeys(_under_wildcard_prefix(model, wildcard_provider_prefix, provider) for model in wildcard_models)
)
def _under_wildcard_prefix(model: str, wildcard_provider_prefix: str, listed_provider: str) -> str:
if model.startswith(f"{wildcard_provider_prefix}/"):
return model
leading, sep, model_suffix = model.partition("/")
leading_is_listed_providers_own_prefix: Final = (
sep != ""
and leading in LLM_PROVIDER_PREFIXES
and (leading == listed_provider or listed_provider.startswith(f"{leading}_"))
)
return f"{wildcard_provider_prefix}/{model_suffix if leading_is_listed_providers_own_prefix else model}"
def expand_wildcard_deployments_for_model_info(

View file

@ -3206,7 +3206,7 @@ class ProxyBaseLLMRequestProcessing:
request_data=self.data,
request=request,
)
if route_type == "aresponses":
if route_type in ("aresponses", "aget_responses"):
selected_data_generator = (
ProxyBaseLLMRequestProcessing._wrap_responses_stream_for_container_ownership(
original_stream_response=response,

View file

@ -102,6 +102,7 @@ class CodexFallbackModel(LiteLLMBaseModel):
experimental_supported_tools: tuple[()] = ()
context_window: int | None
base_instructions: str
supports_search_tool: bool = False
class CodexStockUpgrade(LiteLLMBaseModel):
@ -134,6 +135,7 @@ class CodexStockModel(LiteLLMBaseModel):
upgrade: CodexStockUpgrade | None = None
service_tiers: tuple[CodexServiceTier, ...] = ()
default_service_tier: str | None = None
supports_search_tool: bool = False
supported_reasoning_levels: tuple[CodexReasoningLevel, ...] = ()
context_window: int | None = None
@ -160,6 +162,7 @@ class CodexCatalogRow:
reasoning_effort: str | None = None
display_name: str | None = None
service_tiers: tuple[object, ...] = ()
search_tool_support: tuple[bool | None, ...] = ()
@cache
@ -252,6 +255,12 @@ def configured_service_tiers(
return offered
def offered_search_tool(per_deployment: Sequence[bool | None]) -> bool | None:
if False in per_deployment:
return False
return True if per_deployment and None not in per_deployment else None
def _codex_can_drive(row: CodexCatalogRow) -> bool:
return "*" not in row.id and (row.mode is None or row.mode in CODEX_CHAT_MODES)
@ -306,6 +315,7 @@ def _entry(
{tier.id: tier for tier in (stock_model.service_tiers if stock_model is not None else ())}
)
configured: Final = configured_service_tiers(row.service_tiers, row.id, stock_tiers)
search_tool: Final = offered_search_tool(row.search_tool_support)
if stock_model is None:
return CodexFallbackModel(
slug=row.id,
@ -314,6 +324,7 @@ def _entry(
service_tiers=configured if configured is not None else (),
context_window=row.max_input_tokens,
base_instructions=instructions,
supports_search_tool=search_tool is True,
)
upgrade: Final = (
stock_model.upgrade if stock_model.upgrade is not None and stock_model.upgrade.model in served else None
@ -327,6 +338,7 @@ def _entry(
"visibility": "list",
"supported_in_api": True,
"upgrade": upgrade,
**({} if search_tool is None else {"supports_search_tool": search_tool}),
**(_deployment_defaults(row, stock_model) if is_alias and alias_defaults == "deployment" else {}),
}
if configured is None:
@ -433,6 +445,7 @@ def _catalog_row(
upstream_model=llm_router.get_routable_upstream_model(lookup_id, team_id) if llm_router is not None else None,
display_name=llm_router.get_configured_display_name(label_id) if llm_router is not None else None,
service_tiers=llm_router.get_configured_service_tiers(lookup_id, team_id) if llm_router is not None else (),
search_tool_support=llm_router.get_search_tool_support(lookup_id, team_id) if llm_router is not None else (),
)

View file

@ -31,7 +31,7 @@ if TYPE_CHECKING:
_RESPONSES_API_PROVIDER_PREFIX: Final = "/openai"
_RESPONSES_API_CREATE_ROUTES: Final = frozenset({"/v1/responses", "/responses"})
_RESPONSES_API_ROUTES: Final = frozenset({"/v1/responses", "/responses"})
_ADDRESSED_RESPONSE_ID_FIELD_BY_CALL_TYPE: Final[Mapping[str, str]] = MappingProxyType(
{
"aresponses": "previous_response_id",
@ -104,7 +104,7 @@ def _rewrite_advertised_id(
return event
def _is_responses_api_create_route(request_route: str | None) -> bool:
def _streams_response_ids(request_route: str | None) -> bool:
if request_route is None:
return False
canonical: Final = (
@ -112,7 +112,8 @@ def _is_responses_api_create_route(request_route: str | None) -> bool:
if request_route.startswith(_RESPONSES_API_PROVIDER_PREFIX + "/")
else request_route
)
return canonical in _RESPONSES_API_CREATE_ROUTES
parent, _, response_id = canonical.rpartition("/")
return canonical in _RESPONSES_API_ROUTES or (parent in _RESPONSES_API_ROUTES and response_id != "")
class ResponsesIDSecurity(CustomLogger):
@ -339,7 +340,7 @@ class ResponsesIDSecurity(CustomLogger):
async for chunk in response:
if (
isinstance(chunk, BaseLiteLLMOpenAIResponseObject)
and _is_responses_api_create_route(user_api_key_dict.request_route)
and _streams_response_ids(user_api_key_dict.request_route)
and not general_settings.get("disable_responses_id_security", False)
):
chunk = self._encrypt_response_id(chunk, user_api_key_dict, request_encryption_cache)

View file

@ -4228,5 +4228,33 @@
}
],
"default_model_placeholder": "cursor/claude-4-sonnet"
},
{
"provider": "BIGMODEL",
"provider_display_name": "BigModel (Zhipu AI)",
"litellm_provider": "bigmodel",
"credential_fields": [
{
"key": "api_key",
"label": "BigModel (Zhipu AI) API Key",
"placeholder": null,
"tooltip": null,
"required": true,
"field_type": "password",
"options": null,
"default_value": null
},
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://open.bigmodel.cn",
"tooltip": null,
"required": false,
"field_type": "text",
"options": null,
"default_value": "https://open.bigmodel.cn"
}
],
"default_model_placeholder": "bigmodel/glm-5.3"
}
]

View file

@ -86,6 +86,7 @@ from litellm.proxy.route_llm_request import raise_if_required_body_param_missing
from litellm.proxy.utils import count_request_input_tokens, handle_exception_on_proxy
from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction
from litellm.responses.litellm_completion_transformation.count_request import MODEL_FILE_ID_MAPPING_FIELD
from litellm.responses.retrieve_stream_query import RETRIEVE_INCLUDE_QUERY_KEYS, RETRIEVE_STREAM_QUERY_KEYS
from litellm.responses.streaming_iterator import GuardedResponsesWsFrame, ResponsesWsFrameGuard, spawn_logging_task
from litellm.router import Router
from litellm.types.llms.base import LiteLLMBaseModel
@ -106,11 +107,43 @@ router: Final = APIRouter()
_RESPONSES_WS_CONFIG_VALUE_ADAPTER: Final[TypeAdapter[object | None]] = TypeAdapter(object | None)
_DECODED_JSON: Final = TypeAdapter(object)
_PROXY_SERVER_REQUEST_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
_RETRIEVE_QUERY_PARAMS: Final = tuple(sorted(RETRIEVE_INCLUDE_QUERY_KEYS | RETRIEVE_STREAM_QUERY_KEYS))
_RETRIEVE_QUERY_OPENAPI: Final = {
"parameters": [
{
"name": "stream",
"in": "query",
"required": False,
"schema": {"type": "boolean"},
"description": "Replay the response's events as server-sent events (OpenAI, Azure, and Azure AI Foundry)",
},
{
"name": "starting_after",
"in": "query",
"required": False,
"schema": {"type": "integer", "minimum": 0},
"description": "With stream=true, replay only the events after this sequence number",
},
{
"name": "include",
"in": "query",
"required": False,
"schema": {"type": "array", "items": {"type": "string"}},
"description": "Additional output data to include, forwarded to the provider",
},
{
"name": "include_obfuscation",
"in": "query",
"required": False,
"schema": {"type": "boolean"},
"description": "With stream=true, whether delta events carry obfuscation padding",
},
]
}
_ResponseDocSchemas: TypeAlias = dict[int | str, dict[str, object]] # fastapi's responses kwarg
RESPONSES_API_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = {200: {"model": ResponsesAPIResponse}}
RESPONSES_API_CREATE_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = {
RESPONSES_API_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = {
200: {
"model": ResponsesAPIResponse,
"content": {
@ -254,19 +287,19 @@ async def _resolve_cursor_model_variant_before_auth(request: Request) -> None:
"/v1/responses",
dependencies=[Depends(user_api_key_auth)],
tags=["responses"],
responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS,
responses=RESPONSES_API_RESPONSE_SCHEMAS,
)
@router.post(
"/responses",
dependencies=[Depends(user_api_key_auth)],
tags=["responses"],
responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS,
responses=RESPONSES_API_RESPONSE_SCHEMAS,
)
@router.post(
"/openai/v1/responses",
dependencies=[Depends(user_api_key_auth)],
tags=["responses"],
responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS,
responses=RESPONSES_API_RESPONSE_SCHEMAS,
)
async def responses_api(
request: Request,
@ -749,18 +782,21 @@ async def cursor_chat_completions(
dependencies=[Depends(user_api_key_auth)],
tags=["responses"],
responses=RESPONSES_API_RESPONSE_SCHEMAS,
openapi_extra=_RETRIEVE_QUERY_OPENAPI,
)
@router.get(
"/responses/{response_id}",
dependencies=[Depends(user_api_key_auth)],
tags=["responses"],
responses=RESPONSES_API_RESPONSE_SCHEMAS,
openapi_extra=_RETRIEVE_QUERY_OPENAPI,
)
@router.get(
"/openai/v1/responses/{response_id}",
dependencies=[Depends(user_api_key_auth)],
tags=["responses"],
responses=RESPONSES_API_RESPONSE_SCHEMAS,
openapi_extra=_RETRIEVE_QUERY_OPENAPI,
)
async def get_response(
response_id: str,
@ -785,6 +821,10 @@ async def get_response(
# Get provider response
curl -X GET http://localhost:4000/v1/responses/resp_abc123 \
-H "Authorization: Bearer $LITELLM_MASTER_KEY"
# Resume a background response's event stream after sequence number 7
curl -N "http://localhost:4000/v1/responses/resp_abc123?stream=true&starting_after=7" \
-H "Authorization: Bearer $LITELLM_MASTER_KEY"
```
"""
from litellm.proxy.proxy_server import (
@ -832,6 +872,14 @@ async def get_response(
data: Final = await read_request_body(request=request)
read_settles_billing: Final = not carries_client_side_overrides(data)
data["response_id"] = response_id
retrieve_query: Final = {
key: values[0] if len(values) == 1 else values
for key in _RETRIEVE_QUERY_PARAMS
if (values := request.query_params.getlist(key))
}
if retrieve_query:
body_extra_query: Final[object] = data.get("extra_query")
data["extra_query"] = {**(body_extra_query if isinstance(body_extra_query, dict) else {}), **retrieve_query}
processor: Final = ProxyBaseLLMRequestProcessing(data=data)
try:
response: Final[object] = await processor.base_process_llm_request(
@ -843,7 +891,12 @@ async def get_response(
llm_router=llm_router,
general_settings=general_settings,
proxy_config=proxy_config,
select_data_generator=select_data_generator,
select_data_generator=partial(
_streamed_retrieval_data_generator,
partial(select_data_generator, responses_stream_errors=True),
response_id,
read_settles_billing,
),
model=None,
user_model=user_model,
user_temperature=user_temperature,
@ -917,12 +970,24 @@ async def _bill_finished_background_response(
fastapi_response: Response,
read_settles_billing: bool,
) -> None:
_report_charge(
await _settle_finished_background_read(response, response_id, user_api_key_dict, read_settles_billing),
fastapi_response,
)
async def _settle_finished_background_read(
response: object,
response_id: str,
user_api_key_dict: UserAPIKeyAuth,
read_settles_billing: bool,
) -> "_BilledCharge | None":
from litellm.proxy.proxy_server import proxy_logging_obj
if not isinstance(response, ResponsesAPIResponse) or not is_background_response(response):
return
return None
if response.status not in TERMINAL_RESPONSE_STATUSES:
return
return None
managed_files_hook: Final = _managed_files_hook(proxy_logging_obj)
track_untracked: Final = (
partial(
@ -933,12 +998,11 @@ async def _bill_finished_background_response(
if managed_files_hook is not None
else None
)
charge: Final = await _settle_background_billing(
return await _settle_background_billing(
(lambda cost_check: cost_check.bill_if_finished(response_id, response, track_untracked))
if read_settles_billing
else (lambda cost_check: cost_check.bill_from_deployment(response_id, track_untracked))
)
_report_charge(charge, fastapi_response)
class _BilledCharge(Protocol):
@ -1088,27 +1152,53 @@ def _streamed_background_data_generator(
user_api_key_dict=user_api_key_dict,
request_tags=_background_response_spend_tags(request_data),
)
return _bill_streamed_background_response(response, chunks, track_untracked)
return _bill_streamed_background_response(
response, chunks, partial(_bill_finished_stream, track_untracked=track_untracked)
)
def _streamed_retrieval_data_generator(
select_data_generator: Callable[..., AsyncIterator[str | bytes]],
response_id: str,
read_settles_billing: bool,
response: object,
user_api_key_dict: UserAPIKeyAuth,
request_data: Mapping[str, object],
request: Request | None = None,
) -> AsyncIterator[str | bytes]:
chunks: Final = select_data_generator(
response=response, user_api_key_dict=user_api_key_dict, request_data=request_data, request=request
)
return _bill_streamed_background_response(
response,
chunks,
partial(
_settle_finished_background_read,
response_id=response_id,
user_api_key_dict=user_api_key_dict,
read_settles_billing=read_settles_billing,
),
)
async def _bill_streamed_background_response(
stream: object,
chunks: AsyncIterator[str | bytes],
track_untracked: Callable[[str, ResponsesAPIResponse], Awaitable[bool]],
bill: Callable[[ResponsesAPIResponse], Awaitable[object]],
) -> AsyncIterator[str | bytes]:
async for chunk in chunks:
completed: object = ProxyBaseLLMRequestProcessing.extract_completed_responses_response(stream)
if not isinstance(completed, ResponsesAPIResponse):
yield chunk
continue
await _bill_finished_stream(completed, track_untracked)
await bill(completed)
yield chunk
async for remaining_chunk in chunks:
yield remaining_chunk
return
late_completed: Final = ProxyBaseLLMRequestProcessing.extract_completed_responses_response(stream)
if isinstance(late_completed, ResponsesAPIResponse):
await _bill_finished_stream(late_completed, track_untracked)
await bill(late_completed)
async def _bill_finished_stream(

View file

@ -31,7 +31,12 @@ from litellm.types.realtime import (
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import CallTypes, LlmProviders
from litellm.utils import ProviderConfigManager, resolve_api_key_secret_reference
from litellm.utils import (
ModelInfoLookupMiss,
ProviderConfigManager,
lookup_model_info_for_registration,
resolve_api_key_secret_reference,
)
from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params
from ..litellm_core_utils.credential_accessor import CredentialAccessor
@ -42,8 +47,13 @@ from ..llms.azure.common_utils import get_azure_ad_token
from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client
from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_committed
from ..llms.custom_httpx.http_handler import realtime_ssl_for_url
from ..llms.openai.common_utils import is_openai_backed_api_base
from ..llms.openai.live.handler import OpenAILiveSessions
from ..llms.openai.realtime.handler import OpenAIRealtime
from ..llms.openai.realtime.handler import (
OpenAIRealtime,
confirm_gateway_session_started,
confirm_session_started,
)
from ..llms.openai.workload_identity import openai_static_api_key, resolve_openai_bearer_token_async
from ..llms.vertex_ai.audio_transcription.realtime_transformation import is_vertex_speech_to_text_model
from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig, vertex_realtime_config
@ -740,6 +750,18 @@ def _is_transcription_only_realtime_model(model: str, custom_llm_provider: str)
return "/v1/realtime/transcription_sessions" in (model_info.get("supported_endpoints") or ())
_REALTIME_SERVED_ENDPOINTS: Final = ("/v1/realtime", "/v1/audio/transcriptions", "/v1/live")
_GATEWAY_FIRST_EVENT_WAIT_SECONDS: Final = 3.0
def _cost_map_lists_outside_realtime(model: str) -> bool:
model_info: Final = lookup_model_info_for_registration(model, "openai")
endpoints: Final = (
() if isinstance(model_info, ModelInfoLookupMiss) else model_info.get("supported_endpoints") or ()
)
return bool(endpoints) and not any(endpoint.startswith(_REALTIME_SERVED_ENDPOINTS) for endpoint in endpoints)
_TRANSCRIPTION_QUERY_PARAMS: Final[RealtimeQueryParams] = {"intent": "transcription"}
@ -785,6 +807,7 @@ async def realtime_health_check(
api_version: str | None = None,
realtime_protocol: str | None = None,
model_params: Mapping[str, object] | None = None,
gateway_first_event_wait_seconds: float = _GATEWAY_FIRST_EVENT_WAIT_SECONDS,
) -> bool:
"""
Health check for realtime API - tries connection to the realtime API websocket
@ -800,9 +823,16 @@ async def realtime_health_check(
without the OpenAI-Beta header is bridged to, with transcription-only models adding intent=transcription
Returns:
bool - True if connection is successful, False otherwise
bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an
error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports
the failure in its first server event. Behind any other OpenAI api_base the first event is read for
`gateway_first_event_wait_seconds` only, and a gateway that stays silent that long counts as healthy.
An `invalid_model` error also counts as healthy unless litellm's
cost map lists only endpoints realtime never serves for the model: OpenAI checks the key first, and Live
and transcription models are served through other endpoints. The listed `mode` is not read, since a
deployment's own `mode: realtime` replaces it when the router registers the deployment
Raises:
Exception - if the connection is not successful
Exception - if the connection is not successful or if OpenAI's first server event is any other error
"""
import websockets
@ -822,10 +852,11 @@ async def realtime_health_check(
str | None, api_version or resolved_params.get("api_version")
)
url: str | None = None
openai_api_base: Final = resolved_api_base or litellm.api_base or "https://api.openai.com/"
health_check_api_key: Final = (
await resolve_openai_bearer_token_async(
static_api_key=openai_static_api_key(resolved_api_key),
api_base=resolved_api_base,
api_base=openai_api_base,
litellm_params=resolved_params,
)
if custom_llm_provider == LlmProviders.OPENAI.value
@ -850,10 +881,7 @@ async def realtime_health_check(
query_params=azure_query_params,
)
elif custom_llm_provider == "openai":
url = openai_realtime.construct_url(
api_base=resolved_api_base or "https://api.openai.com/",
query_params={"model": model},
)
url = openai_realtime.construct_url(api_base=openai_api_base, query_params={"model": model})
elif custom_llm_provider == "xai":
url = xai_realtime.construct_url(
api_base=resolved_api_base or "https://api.x.ai/v1", query_params={"model": model}
@ -899,8 +927,23 @@ async def realtime_health_check(
additional_headers=auth_headers,
max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
ssl=realtime_ssl_for_url(url),
):
return True
) as connection:
if custom_llm_provider != "openai":
return True
if not is_openai_backed_api_base(url):
return await confirm_gateway_session_started(
connection,
model,
sent_credentials=bool(auth_headers),
model_listed_outside_realtime=_cost_map_lists_outside_realtime(model),
first_event_wait_seconds=gateway_first_event_wait_seconds,
)
return await confirm_session_started(
connection,
model,
sent_credentials=bool(auth_headers),
model_listed_outside_realtime=_cost_map_lists_outside_realtime(model),
)
_realtime_health_check = realtime_health_check

View file

@ -1,5 +1,8 @@
from collections.abc import Sequence
import json
from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass
from itertools import groupby
from types import MappingProxyType
from typing import Final, cast # noqa: TID251 # validating the openai tool union strips vendor keys from raw tools
from pydantic import ValidationError
@ -15,8 +18,8 @@ class _InputItemType(LiteLLMBaseModel):
type: str = ""
class _AdditionalToolsItem(LiteLLMBaseModel):
tools: tuple[dict[str, object], ...] = ()
class _ToolList(LiteLLMBaseModel):
tools: tuple[Mapping[str, object], ...] = ()
@dataclass(frozen=True, slots=True)
@ -33,12 +36,54 @@ def _is_additional_tools_item(item: object) -> bool:
return False
def _tools_of_item(item: object) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]:
def _tool_list_of(value: object) -> tuple[Mapping[str, object], ...]:
try:
parsed: Final = _AdditionalToolsItem.model_validate(item)
return _ToolList.model_validate(value).tools
except ValidationError:
return ()
return tuple(cast("ALL_RESPONSES_API_TOOL_PARAMS", tool) for tool in parsed.tools)
def _tools_of_item(item: object) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]:
return tuple(cast("ALL_RESPONSES_API_TOOL_PARAMS", tool) for tool in _tool_list_of(item))
def _tool_identity(tool: Mapping[str, object]) -> str:
name: Final = tool.get("name")
if isinstance(name, str) and name:
return f"{tool.get('type')}:{name}"
return json.dumps(tool, sort_keys=True, default=str)
def _members_of_copies(copies: Sequence[Mapping[str, object]]) -> Iterator[Mapping[str, object]]:
for namespace in copies:
yield from _tool_list_of(namespace)
def _merged(copies: Sequence[Mapping[str, object]]) -> Mapping[str, object]:
newest: Final = copies[-1]
if newest.get("type") != "namespace" or len(copies) == 1:
return newest
members: Final = tuple(_members_of_copies(copies))
return {**newest, "tools": list(_unique_tools(members))} if members else newest
def _identity_of(keyed_tool: tuple[str, Mapping[str, object]]) -> str:
return keyed_tool[0]
def _unique_tools(tools: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]:
"""Codex re-sends its tool list in the input on every turn, so the same tool arrives many times.
Keep each tool once, at its first position, with the newest copy's definition, and merge the
members of a namespace sent more than once."""
keyed: Final = tuple((_tool_identity(tool), tool) for tool in tools)
copies_by_identity: Final = MappingProxyType(
{
identity: tuple(tool for _, tool in group)
for identity, group in groupby(sorted(keyed, key=_identity_of), key=_identity_of)
}
)
first_seen_order: Final = tuple(dict.fromkeys(identity for identity, _ in keyed))
return tuple(_merged(copies_by_identity[identity]) for identity in first_seen_order)
def hoist_additional_tools(
@ -58,4 +103,8 @@ def hoist_additional_tools(
len(items),
)
remaining_input: Final = [item for item in input if not _is_additional_tools_item(item)]
return HoistedAdditionalTools(input=remaining_input, tools=(*existing, *hoisted), hoisted=hoisted)
unique: Final = tuple(
cast("ALL_RESPONSES_API_TOOL_PARAMS", tool) # cast-ok: a merged namespace is rebuilt from the same tool dicts
for tool in _unique_tools((*existing, *hoisted))
)
return HoistedAdditionalTools(input=remaining_input, tools=unique, hoisted=hoisted)

View file

@ -0,0 +1,268 @@
import json
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass
from itertools import chain
from types import MappingProxyType
from typing import Final, cast # noqa: TID251 # loaded tool specs are forwarded as the client sent them
from openai.types.responses.response_input_param import (
FunctionCallOutput,
ResponseFunctionToolCallParam,
ResponseInputItemParam,
)
from pydantic import ConfigDict, TypeAdapter, ValidationError
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import (
ALL_RESPONSES_API_TOOL_PARAMS,
ChatCompletionToolParam,
ChatCompletionToolParamFunctionChunk,
ResponseInputParam,
)
from .custom_tools import function_chat_tool
from .tool_search import CLIENT_TOOL_SEARCH_FUNCTION_NAME, has_client_tool_search
__all__ = [
"ClientToolSearchBridge",
"bridge_client_tool_search",
"client_tool_search_chat_tool",
"client_tool_search_loaded_tools",
]
_DEFAULT_SEARCH_DESCRIPTION: Final = (
"Search the deferred tools by keyword. The matching tools are loaded and become callable on the next turn."
)
_DEFAULT_SEARCH_PARAMETERS: Final = MappingProxyType(
{
"type": "object",
"properties": {
"query": {"type": "string", "description": "Search query for deferred tools."},
"limit": {"type": "integer", "description": "Maximum number of tools to return. Defaults to 8."},
},
"required": ["query"],
"additionalProperties": False,
}
)
class _ClientToolSearchItem(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
type: str = ""
execution: str | None = None
call_id: str | None = None
arguments: object = None
tools: tuple[Mapping[str, object], ...] = ()
class _ToolSpec(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
type: str = ""
name: str = ""
description: str = ""
defer_loading: object = None
tools: tuple[Mapping[str, object], ...] = ()
class _ClientToolSearchTool(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
description: str = ""
parameters: Mapping[str, object] | None = None
@dataclass(frozen=True, slots=True)
class _LoadedTool:
namespace: str | None
namespace_description: str
spec: Mapping[str, object]
@property
def callable_name(self) -> str:
name: Final = str(self.spec.get("name") or "")
return name if self.namespace is None else f"{self.namespace}__{name}"
@dataclass(frozen=True, slots=True)
class ClientToolSearchBridge:
input: str | ResponseInputParam
tools: tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]
bridged: bool
def client_tool_search_chat_tool(tool: Mapping[str, object]) -> ChatCompletionToolParam:
spec: Final = _ClientToolSearchTool.model_validate(tool)
function: Final = ChatCompletionToolParamFunctionChunk(
name=CLIENT_TOOL_SEARCH_FUNCTION_NAME,
description=spec.description or _DEFAULT_SEARCH_DESCRIPTION,
parameters=dict(spec.parameters or _DEFAULT_SEARCH_PARAMETERS),
)
return function_chat_tool(function, None, False)
def _client_tool_search_item(item: object) -> _ClientToolSearchItem | None:
try:
parsed: Final = _ClientToolSearchItem.model_validate(item)
except ValidationError:
return None
if parsed.type not in ("tool_search_call", "tool_search_output") or parsed.execution != "client":
return None
return parsed
_INPUT_ITEMS: Final = TypeAdapter(tuple[object, ...])
def _searched_tools(item: object) -> tuple[Mapping[str, object], ...]:
parsed: Final = _client_tool_search_item(item)
if parsed is None or parsed.type != "tool_search_output" or not parsed.call_id:
return ()
return parsed.tools
def client_tool_search_loaded_tools(input: object) -> tuple[Mapping[str, object], ...]:
if not isinstance(input, (list, tuple)):
return ()
return tuple(chain.from_iterable(map(_searched_tools, _INPUT_ITEMS.validate_python(input))))
def _tool_spec(tool: object) -> _ToolSpec | None:
try:
return _ToolSpec.model_validate(tool)
except ValidationError:
return None
def _as_tool_param(spec: Mapping[str, object]) -> ALL_RESPONSES_API_TOOL_PARAMS:
return cast("ALL_RESPONSES_API_TOOL_PARAMS", spec) # cast-ok: client tool specs are forwarded as sent
def _without_deferral(spec: Mapping[str, object]) -> Mapping[str, object]:
return {key: value for key, value in spec.items() if key != "defer_loading"}
def _loaded_tools(tool: Mapping[str, object]) -> tuple[_LoadedTool, ...]:
spec: Final = _tool_spec(tool)
if spec is None:
return ()
if spec.type != "namespace":
return (_LoadedTool(namespace=None, namespace_description="", spec=_without_deferral(tool)),)
return tuple(
_LoadedTool(namespace=spec.name, namespace_description=spec.description, spec=_without_deferral(member))
for member in spec.tools
)
def _all_loaded_tools(tools: Iterable[Mapping[str, object]]) -> tuple[_LoadedTool, ...]:
return tuple(chain.from_iterable(map(_loaded_tools, tools)))
def _callable_names(tool: object) -> tuple[str, ...]:
spec: Final = _tool_spec(tool)
if spec is None:
return ()
if spec.type != "namespace":
return (spec.name,)
return tuple(f"{spec.name}__{member.get('name') or ''}" for member in spec.tools)
def _first_occurrences(loaded: Sequence[_LoadedTool], taken: frozenset[str]) -> tuple[_LoadedTool, ...]:
first_index: Final = {tool.callable_name: index for index, tool in reversed(tuple(enumerate(loaded)))}
return tuple(
tool
for index, tool in enumerate(loaded)
if first_index[tool.callable_name] == index and tool.callable_name not in taken
)
def _regrouped(loaded: Sequence[_LoadedTool]) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]:
namespaces: Final = {tool.namespace: tool.namespace_description for tool in reversed(loaded) if tool.namespace}
ordered_namespaces: Final = tuple(dict.fromkeys(tool.namespace for tool in loaded if tool.namespace))
top_level: Final = tuple(tool.spec for tool in loaded if tool.namespace is None)
grouped: Final = tuple(
{
"type": "namespace",
"name": namespace,
"description": namespaces[namespace],
"tools": tuple(tool.spec for tool in loaded if tool.namespace == namespace),
}
for namespace in ordered_namespaces
)
return tuple(_as_tool_param(tool) for tool in (*top_level, *grouped))
def _is_deferred(tool: object) -> bool:
spec: Final = _tool_spec(tool)
return spec is not None and spec.defer_loading is True
def _without_deferred(tool: ALL_RESPONSES_API_TOOL_PARAMS) -> ALL_RESPONSES_API_TOOL_PARAMS | None:
spec: Final = _tool_spec(tool)
if spec is None:
return tool
if spec.type != "namespace":
return None if spec.defer_loading is True else tool
eager_members: Final = tuple(member for member in spec.tools if not _is_deferred(member))
if len(eager_members) == len(spec.tools):
return tool
if not eager_members:
return None
return _as_tool_param({**tool, "tools": eager_members})
def _search_output_text(loaded: Sequence[_LoadedTool]) -> str:
if not loaded:
return "No tools matched the search."
return "Loaded these tools, which are now callable: " + ", ".join(tool.callable_name for tool in loaded)
def _search_arguments_json(arguments: object) -> str:
return arguments if isinstance(arguments, str) else json.dumps(arguments or {})
def _bridged_item(item: ResponseInputItemParam) -> ResponseInputItemParam | None:
parsed: Final = _client_tool_search_item(item)
if parsed is None:
return item
if not parsed.call_id:
return None
if parsed.type == "tool_search_call":
return ResponseFunctionToolCallParam(
type="function_call",
call_id=parsed.call_id,
name=CLIENT_TOOL_SEARCH_FUNCTION_NAME,
arguments=_search_arguments_json(parsed.arguments),
)
return FunctionCallOutput(
type="function_call_output", call_id=parsed.call_id, output=_search_output_text(_all_loaded_tools(parsed.tools))
)
def bridge_client_tool_search(
input: str | ResponseInputParam,
tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None,
loaded_earlier: Sequence[Mapping[str, object]] = (),
) -> ClientToolSearchBridge:
existing: Final = tuple(tools or ())
searches_on_client: Final = has_client_tool_search(existing)
items: Final = () if isinstance(input, str) else tuple(input)
search_items: Final = tuple(parsed for item in items if (parsed := _client_tool_search_item(item)) is not None)
if not searches_on_client and not search_items:
return ClientToolSearchBridge(input=input, tools=existing, bridged=False)
eager: Final = (
tuple(kept for tool in existing if (kept := _without_deferred(tool)) is not None)
if searches_on_client
else existing
)
loaded: Final = _all_loaded_tools((*loaded_earlier, *client_tool_search_loaded_tools(input)))
taken: Final = frozenset(chain.from_iterable(map(_callable_names, eager)))
bridged_input: Final = (
input if isinstance(input, str) else [bridged for item in input if (bridged := _bridged_item(item)) is not None]
)
return ClientToolSearchBridge(
input=bridged_input,
tools=(*eager, *_regrouped(_first_occurrences(loaded, taken))),
bridged=True,
)

View file

@ -20,7 +20,7 @@ from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
from litellm.responses.main import previous_response_client_ids
from litellm.types.llms.openai import ResponseInputParam
from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInputParam
from litellm.utils import ProviderConfigManager
_COUNTED_CHAT_PARAMS: Final = (
@ -119,14 +119,13 @@ def _with_hoisted_additional_tools(responses_request: Mapping[str, object]) -> M
if not isinstance(responses_input, list):
return responses_request
input_items: Final = cast(ResponseInputParam, responses_input) # cast-ok: forwarded as sent, as inference does
hoisted: Final = hoist_additional_tools(input_items, None)
top_level_tools: Final = cast( # cast-ok: tools are forwarded as sent, as inference does
"tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]", _ITEMS.validate_python(responses_request.get("tools") or ())
)
hoisted: Final = hoist_additional_tools(input_items, top_level_tools)
if not hoisted.hoisted:
return responses_request
return {
**responses_request,
"input": hoisted.input,
"tools": [*_ITEMS.validate_python(responses_request.get("tools") or ()), *hoisted.hoisted],
}
return {**responses_request, "input": hoisted.input, "tools": list(hoisted.tools)}
def _with_emulated_file_search(responses_request: Mapping[str, object]) -> Mapping[str, object]:

View file

@ -2,17 +2,21 @@
Handler for transforming responses api requests to litellm.completion requests
"""
from collections.abc import Coroutine, Mapping
from collections.abc import Coroutine, Mapping, Sequence
from dataclasses import dataclass
from typing import Final
from pydantic import TypeAdapter
import litellm
from litellm.responses.additional_tools import hoist_additional_tools
from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools
from litellm.responses.litellm_completion_transformation.client_tool_search import bridge_client_tool_search
from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary
from litellm.responses.litellm_completion_transformation.session_handler import ResponsesSessionHandler, StoredTurns
from litellm.responses.litellm_completion_transformation.streaming_iterator import (
LiteLLMCompletionStreamingIterator,
)
from litellm.responses.litellm_completion_transformation.tool_search import has_client_tool_search
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
with_compaction_items_as_params,
@ -28,6 +32,28 @@ from litellm.types.utils import ModelResponse
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object])
@dataclass(frozen=True, slots=True)
class _BridgedRequest:
input: str | ResponseInputParam
responses_api_request: ResponsesAPIOptionalRequestParams
def _bridged_request(
hoisted: HoistedAdditionalTools,
responses_api_request: ResponsesAPIOptionalRequestParams,
loaded_earlier: Sequence[Mapping[str, object]],
) -> _BridgedRequest:
searched: Final = bridge_client_tool_search(hoisted.input, hoisted.tools, loaded_earlier)
return _BridgedRequest(
input=searched.input,
responses_api_request=(
{**responses_api_request, "tools": list(searched.tools)}
if hoisted.hoisted or searched.bridged
else responses_api_request
),
)
class LiteLLMCompletionTransformationHandler:
def response_api_handler(
self,
@ -48,10 +74,22 @@ class LiteLLMCompletionTransformationHandler:
| Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]
):
hoisted: Final = hoist_additional_tools(input, responses_api_request.get("tools"))
bridged_input: Final = hoisted.input
bridged_request: Final[ResponsesAPIOptionalRequestParams] = (
{**responses_api_request, "tools": list(hoisted.tools)} if hoisted.hoisted else responses_api_request
)
previous_response_id: Final = responses_api_request.get("previous_response_id")
if _is_async and previous_response_id and has_client_tool_search(hoisted.tools):
return self._async_response_api_handler_after_earlier_searches(
previous_response_client_ids,
previous_response_id,
hoisted,
model=model,
responses_api_request=responses_api_request,
custom_llm_provider=custom_llm_provider,
stream=stream,
extra_headers=extra_headers,
request_kwargs=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs),
)
bridged: Final = _bridged_request(hoisted, responses_api_request, ())
bridged_input: Final = bridged.input
bridged_request: Final = bridged.responses_api_request
litellm_completion_request: Final[dict] = (
LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
model=model,
@ -73,13 +111,8 @@ class LiteLLMCompletionTransformationHandler:
**kwargs,
)
completion_args: Final = {}
completion_args.update(kwargs)
completion_args.update(litellm_completion_request)
completion_args["_skip_responses_api_bridge"] = True
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion(
**completion_args,
**{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True},
)
if isinstance(litellm_completion_response, ModelResponse):
@ -104,9 +137,53 @@ class LiteLLMCompletionTransformationHandler:
)
raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")
async def _async_response_api_handler_after_earlier_searches(
self,
previous_response_client_ids: tuple[str, ...],
previous_response_id: str,
hoisted: HoistedAdditionalTools,
/,
*,
model: str,
responses_api_request: ResponsesAPIOptionalRequestParams,
custom_llm_provider: str | None,
stream: bool | None,
extra_headers: Mapping[str, object] | None,
request_kwargs: Mapping[str, object],
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
stored_turns: Final = await ResponsesSessionHandler.get_stored_turns_for_previous_response_id(
previous_response_id, previous_response_client_ids
)
bridged: Final = _bridged_request(
hoisted,
responses_api_request,
ResponsesSessionHandler.client_tool_search_tools_loaded_before(
stored_turns, previous_response_id, previous_response_client_ids
),
)
return await self.async_response_api_handler(
previous_response_client_ids,
stored_turns,
litellm_completion_request=(
LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
model=model,
input=bridged.input,
responses_api_request=bridged.responses_api_request,
custom_llm_provider=custom_llm_provider,
stream=stream,
extra_headers=extra_headers,
**request_kwargs,
)
),
request_input=bridged.input,
responses_api_request=bridged.responses_api_request,
**request_kwargs,
)
async def async_response_api_handler(
self,
previous_response_client_ids: tuple[str, ...] = (),
stored_turns: StoredTurns | None = None,
/,
*,
litellm_completion_request: dict,
@ -122,16 +199,12 @@ class LiteLLMCompletionTransformationHandler:
instructions=responses_api_request.get("instructions"),
request_kwargs=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs),
previous_response_client_ids=previous_response_client_ids,
stored_turns=stored_turns,
input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(request_input)),
)
acompletion_args: Final = {}
acompletion_args.update(kwargs)
acompletion_args.update(litellm_completion_request)
acompletion_args["_skip_responses_api_bridge"] = True
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion(
**acompletion_args,
**{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True},
)
if isinstance(litellm_completion_response, ModelResponse):
responses_api_response: Final[ResponsesAPIResponse] = (

View file

@ -9,6 +9,10 @@ import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER
from litellm.litellm_core_utils.cold_storage_handler import ColdStorageHandler
from litellm.responses.litellm_completion_transformation.client_tool_search import (
bridge_client_tool_search,
client_tool_search_loaded_tools,
)
from litellm.responses.litellm_completion_transformation.compaction_items import (
holds_compaction_summary,
input_items,
@ -88,6 +92,12 @@ class _ReplayStart:
at_output: bool
@dataclass(frozen=True, slots=True)
class StoredTurns:
spend_logs: tuple["SpendLogsPayload", ...]
proxy_server_requests: tuple[Mapping[str, object] | None, ...]
def _compaction_point(
index: int, spend_log: "SpendLogsPayload", proxy_server_request: Mapping[str, object] | None
) -> _ReplayStart | None:
@ -190,6 +200,7 @@ class ResponsesSessionHandler:
previous_response_id: str,
item_cache_control: bool = False,
client_response_ids: tuple[str, ...] = (),
stored_turns: StoredTurns | None = None,
) -> ChatCompletionSession:
"""
Return the chat completion message history for a previous response id
@ -200,23 +211,21 @@ class ResponsesSessionHandler:
)
verbose_proxy_logger.debug("inside get_chat_completion_message_history_for_previous_response_id")
all_spend_logs: Final = [
spend_log
for spend_log in await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id(
turns: Final = (
stored_turns
if stored_turns is not None
else await ResponsesSessionHandler.get_stored_turns_for_previous_response_id(
previous_response_id, client_response_ids
)
if _is_a_conversation_turn(spend_log)
]
)
all_spend_logs: Final = turns.spend_logs
verbose_proxy_logger.debug("found %s spend logs for this response id", len(all_spend_logs))
litellm_session_id: str | None = None
if len(all_spend_logs) > 0:
litellm_session_id = all_spend_logs[0].get("session_id")
proxy_server_requests: Final = [
await ResponsesSessionHandler.get_proxy_server_request_from_spend_log(spend_log=spend_log)
for spend_log in all_spend_logs
]
proxy_server_requests: Final = turns.proxy_server_requests
replayable_turns: Final = _turns_through_referenced_response(
all_spend_logs, _referenced_request_ids(previous_response_id, client_response_ids)
)
@ -252,6 +261,45 @@ class ResponsesSessionHandler:
instructions=_latest_replayable_instructions(proxy_server_requests),
)
@staticmethod
async def get_stored_turns_for_previous_response_id(
previous_response_id: str,
client_response_ids: tuple[str, ...] = (),
) -> StoredTurns:
spend_logs: Final = tuple(
spend_log
for spend_log in await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id(
previous_response_id, client_response_ids
)
if _is_a_conversation_turn(spend_log)
)
return StoredTurns(
spend_logs=spend_logs,
proxy_server_requests=tuple(
[
await ResponsesSessionHandler.get_proxy_server_request_from_spend_log(spend_log=spend_log)
for spend_log in spend_logs
]
),
)
@staticmethod
def client_tool_search_tools_loaded_before(
stored_turns: StoredTurns,
previous_response_id: str,
client_response_ids: tuple[str, ...] = (),
) -> tuple[Mapping[str, object], ...]:
turns: Final = _turns_through_referenced_response(
stored_turns.spend_logs, _referenced_request_ids(previous_response_id, client_response_ids)
)
return tuple(
chain.from_iterable(
client_tool_search_loaded_tools(request.get("input"))
for request in stored_turns.proxy_server_requests[:turns]
if request is not None
)
)
@staticmethod
def _history_of_turns(
spend_logs: Sequence["SpendLogsPayload"],
@ -310,7 +358,7 @@ class ResponsesSessionHandler:
if response_input_param:
chat_completion_message_history.extend(
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=response_input_param,
input=bridge_client_tool_search(response_input_param, None).input,
responses_api_request={},
replay_reasoning=True,
item_cache_control=item_cache_control,
@ -325,7 +373,7 @@ class ResponsesSessionHandler:
if (responses_api_output := _stored_responses_api_output(_response_output)) is not None:
chat_completion_message_history.extend(
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=responses_api_output,
input=bridge_client_tool_search(responses_api_output, None).input,
responses_api_request={},
replay_reasoning=True,
item_cache_control=item_cache_control,

View file

@ -1,7 +1,8 @@
import time
import uuid
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from itertools import filterfalse
from types import MappingProxyType
from typing import Any, Final, cast
import litellm
@ -18,9 +19,9 @@ from litellm.responses.litellm_completion_transformation.custom_tools import (
)
from litellm.responses.litellm_completion_transformation.reasoning_items import mint_reasoning_item_id
from litellm.responses.litellm_completion_transformation.tool_search import (
ToolSearchExecution,
build_tool_search_call,
has_server_tool_search,
is_anthropic_tool_search_use,
tool_search_routing,
)
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
@ -192,16 +193,21 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
self._namespace_tool_names = LiteLLMCompletionResponsesConfig.namespace_tool_name_map(request_tools)
self._web_search_calls: dict[str, object] = {} # mutable-ok: latest call by provider id
self._queued_web_search_call_ids: set[str] = set() # mutable-ok: emitted call ids
self._tool_search_requested: bool = has_server_tool_search(request_tools)
self._tool_search_call_ids: frozenset[str] = frozenset()
self._tool_search = tool_search_routing(request_tools)
self._tool_search_executions: Mapping[str, ToolSearchExecution] = MappingProxyType({})
def _is_tool_search_call(self, call_id: str, fn_name: str) -> bool:
if call_id in self._tool_search_call_ids:
return True
if not self._tool_search_requested or not is_anthropic_tool_search_use(call_id, fn_name):
return False
self._tool_search_call_ids = self._tool_search_call_ids | {call_id}
return True
async def aclose(self) -> None:
await self.litellm_custom_stream_wrapper.aclose()
def _tool_search_execution(self, call_id: str, fn_name: str) -> ToolSearchExecution | None:
known: Final = self._tool_search_executions.get(call_id)
if known is not None:
return known
execution: Final = self._tool_search.execution_of(call_id, fn_name)
if execution is None:
return None
self._tool_search_executions = MappingProxyType({**self._tool_search_executions, call_id: execution})
return execution
def _next_output_item_added(self, output_index: int, item: BaseLiteLLMOpenAIResponseObject) -> OutputItemAddedEvent:
self._sequence_number += 1
@ -216,18 +222,22 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
def _queue_output_item_added(self, output_index: int, item: BaseLiteLLMOpenAIResponseObject) -> None:
self._pending_tool_events.append(self._next_output_item_added(output_index, item))
def _queue_tool_search_call_added(self, call_id: str, output_index: int, args_delta: str) -> None:
def _queue_tool_search_call_added(
self, call_id: str, output_index: int, args_delta: str, execution: ToolSearchExecution
) -> None:
if call_id not in self._tool_args_by_call_id:
self._tool_args_by_call_id[call_id] = ""
item: Final = build_tool_search_call(call_id, "", "in_progress")
item: Final = build_tool_search_call(call_id, "", "in_progress", execution)
self._tool_item_id_by_call_id[call_id] = item.id
self._queue_output_item_added(output_index, BaseLiteLLMOpenAIResponseObject(**item.model_dump()))
self._tool_args_by_call_id[call_id] += args_delta
def _queue_tool_search_call_done(self, call_id: str, output_index: int, final_args: str) -> None:
self._queue_tool_search_call_added(call_id, output_index, "")
def _queue_tool_search_call_done(
self, call_id: str, output_index: int, final_args: str, execution: ToolSearchExecution
) -> None:
self._queue_tool_search_call_added(call_id, output_index, "", execution)
item: Final = build_tool_search_call(
call_id, final_args or self._tool_args_by_call_id.get(call_id, ""), "completed"
call_id, final_args or self._tool_args_by_call_id.get(call_id, ""), "completed", execution
)
self._sequence_number += 1
self._pending_tool_events.append(
@ -436,8 +446,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
self._queue_held_annotations()
output_index = self._get_or_assign_tool_output_index(call_id)
if self._is_tool_search_call(call_id, fn_name):
self._queue_tool_search_call_added(call_id, output_index, fn_args_delta)
if (search_execution := self._tool_search_execution(call_id, fn_name)) is not None:
self._queue_tool_search_call_added(call_id, output_index, fn_args_delta, search_execution)
continue
self._started_tool_call = True
@ -505,8 +515,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
self._queue_web_search_events(call_id, web_search_call)
self._queued_web_search_call_ids.add(call_id)
continue
if self._is_tool_search_call(call_id, fn_name):
self._queue_tool_search_call_done(call_id, output_index, fn_args)
if (search_execution := self._tool_search_execution(call_id, fn_name)) is not None:
self._queue_tool_search_call_done(call_id, output_index, fn_args, search_execution)
continue
# Track if this is a new tool call that wasn't streamed

View file

@ -1,4 +1,5 @@
from collections.abc import Iterable, Iterator, Mapping, Sequence, Set
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
from pydantic import ConfigDict, TypeAdapter, ValidationError
@ -18,25 +19,31 @@ if TYPE_CHECKING:
__all__ = [
"ANTHROPIC_TOOL_SEARCH_TOOL_NAME",
"ANTHROPIC_TOOL_SEARCH_TOOL_TYPE",
"CLIENT_TOOL_SEARCH_FUNCTION_NAME",
"CompletionRoute",
"ToolSearchCallStatus",
"ToolSearchExecution",
"ToolSearchRouting",
"answers_tool_search",
"anthropic_tool_search_chat_tool",
"build_tool_search_call",
"chat_config_builds_anthropic_request",
"chat_messages_without_tool_search_calls",
"chat_route_rejects_unanswered_tool_calls",
"has_client_tool_search",
"has_server_tool_search",
"holds_tool_search_use",
"input_without_unanswered_anthropic_tool_search_calls",
"is_anthropic_tool_search_chat_tool",
"is_anthropic_tool_search_use",
"is_client_tool_search_tool",
"is_openai_tool_search_tool",
"is_server_tool_search_tool",
"provider_builds_anthropic_request",
"provider_chat_config",
"runs_tool_search_on_the_server",
"tool_search_call_ids",
"tool_search_routing",
"tools_with_anthropic_tool_search_mapped",
"tools_without_anthropic_tool_search",
"unanswered_tool_search_call_ids",
@ -46,7 +53,9 @@ __all__ = [
ANTHROPIC_TOOL_SEARCH_TOOL_TYPE: Final = "tool_search_tool_regex_20251119"
ANTHROPIC_TOOL_SEARCH_TOOL_NAME: Final = "tool_search_tool_regex"
CLIENT_TOOL_SEARCH_FUNCTION_NAME: Final = "tool_search"
ToolSearchCallStatus: TypeAlias = Literal["in_progress", "completed", "incomplete"]
ToolSearchExecution: TypeAlias = Literal["server", "client"]
_ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object])
_MESSAGE_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
@ -79,6 +88,35 @@ def has_server_tool_search(tools: Sequence[object] | None) -> bool:
return any(is_server_tool_search_tool(tool) for tool in tools or ())
def is_client_tool_search_tool(tool: object) -> bool:
try:
parsed: Final = _ToolSearchFields.model_validate(tool)
except ValidationError:
return False
return parsed.type == "tool_search" and parsed.execution == "client"
def has_client_tool_search(tools: Sequence[object] | None) -> bool:
return any(is_client_tool_search_tool(tool) for tool in tools or ())
@dataclass(frozen=True, slots=True)
class ToolSearchRouting:
server: bool
client: bool
def execution_of(self, tool_id: str, tool_name: str) -> ToolSearchExecution | None:
if self.server and is_anthropic_tool_search_use(tool_id, tool_name):
return "server"
if self.client and tool_name == CLIENT_TOOL_SEARCH_FUNCTION_NAME:
return "client"
return None
def tool_search_routing(tools: Sequence[object] | None) -> ToolSearchRouting:
return ToolSearchRouting(server=has_server_tool_search(tools), client=has_client_tool_search(tools))
def anthropic_tool_search_chat_tool() -> AnthropicToolSearchToolRegex:
return AnthropicToolSearchToolRegex(type=ANTHROPIC_TOOL_SEARCH_TOOL_TYPE, name=ANTHROPIC_TOOL_SEARCH_TOOL_NAME)
@ -346,13 +384,18 @@ def _parsed_arguments(arguments_json: str) -> Mapping[str, object] | None:
return None
def build_tool_search_call(tool_id: str, arguments_json: str, status: ToolSearchCallStatus) -> OutputToolSearchCall:
def build_tool_search_call(
tool_id: str,
arguments_json: str,
status: ToolSearchCallStatus,
execution: ToolSearchExecution = "server",
) -> OutputToolSearchCall:
arguments: Final = _parsed_arguments(arguments_json)
return OutputToolSearchCall(
id=openai_shaped_tool_call_item_id("tool_search_call", tool_id),
call_id=tool_id,
arguments={} if arguments is None else arguments,
execution="server",
execution=execution,
status="incomplete" if arguments is None else status,
type="tool_search_call",
)

View file

@ -64,6 +64,7 @@ from litellm.llms.snowflake.chat.transformation import is_snowflake_claude_model
from litellm.responses.bridge_check import chat_target_accepts_reasoning_summary
from litellm.responses.litellm_completion_transformation.session_handler import (
ResponsesSessionHandler,
StoredTurns,
)
from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex
from litellm.types.llms.base import CachedTokensDetails
@ -124,6 +125,7 @@ from litellm.types.utils import (
Usage,
)
from .client_tool_search import client_tool_search_chat_tool
from .compaction_items import (
carries_only_compaction,
compaction_output_item,
@ -151,16 +153,17 @@ from .custom_tools import (
from .reasoning_items import decode_thinking_blocks, encode_thinking_blocks, mint_reasoning_item_id
from .tool_search import (
CompletionRoute,
ToolSearchRouting,
answers_tool_search,
anthropic_tool_search_chat_tool,
build_tool_search_call,
has_server_tool_search,
holds_tool_search_use,
is_anthropic_tool_search_chat_tool,
is_anthropic_tool_search_use,
is_server_tool_search_tool,
provider_builds_anthropic_request,
tool_search_call_ids,
tool_search_routing,
without_tool_search_calls,
without_tool_search_uses,
)
@ -1342,6 +1345,7 @@ class LiteLLMCompletionResponsesConfig:
request_kwargs: Mapping[str, object] = MappingProxyType({}),
previous_response_client_ids: tuple[str, ...] = (),
input_replaces_history: bool = False,
stored_turns: StoredTurns | None = None,
) -> dict[str, object]:
"""
Async hook to get the chain of previous input and output pairs and return a list of Chat Completion messages
@ -1357,6 +1361,7 @@ class LiteLLMCompletionResponsesConfig:
ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id,
item_cache_control=keeps_item_marks,
client_response_ids=previous_response_client_ids,
stored_turns=stored_turns,
)
chat_completion_session: Final = (
await lookup(previous_response_id)
@ -2818,9 +2823,11 @@ class LiteLLMCompletionResponsesConfig:
if tool_type == "custom":
converted: Final = convert_custom_tool_to_function_tool(tool)
return ResponsesToolChatForm(chat_tools=() if converted is None else (converted,), web_search_options=None)
if tool_type == "tool_search" and validated_tool_search_execution(tool.get("execution")) != "client":
if tool_type == "tool_search" and validated_tool_search_execution(tool.get("execution")) == "client":
return ResponsesToolChatForm(chat_tools=(client_tool_search_chat_tool(tool),), web_search_options=None)
if tool_type == "tool_search":
return ResponsesToolChatForm(chat_tools=(anthropic_tool_search_chat_tool(),), web_search_options=None)
if tool_type in ("computer_use", "image_generation", "local_shell", "shell", "tool_search"):
if tool_type in ("computer_use", "image_generation", "local_shell", "shell"):
verbose_logger.warning(
"Dropping Responses API tool of type '%s': it has no Chat Completions "
"equivalent and the target provider would reject the request.",
@ -2960,7 +2967,7 @@ class LiteLLMCompletionResponsesConfig:
request_tools: Final = responses_api_request.get("tools") if responses_api_request is not None else None
custom_tool_names: Final = extract_custom_tool_names(request_tools)
namespace_tool_names: Final = LiteLLMCompletionResponsesConfig.namespace_tool_name_map(request_tools)
tool_search_requested: Final = has_server_tool_search(request_tools)
tool_search: Final = tool_search_routing(request_tools)
web_search_calls: Final = LiteLLMCompletionResponsesConfig._web_search_calls_by_call_id(
chat_completion_response
@ -2977,7 +2984,7 @@ class LiteLLMCompletionResponsesConfig:
if (
server_tool_call := LiteLLMCompletionResponsesConfig._server_tool_call_item(
tool_id, tool_name, tool_arguments, web_search_calls, tool_search_requested
tool_id, tool_name, tool_arguments, web_search_calls, tool_search
)
) is not None:
responses_tools.append(server_tool_call)
@ -3042,14 +3049,15 @@ class LiteLLMCompletionResponsesConfig:
tool_name: str,
tool_arguments: str,
web_search_calls: Mapping[str, ResponseFunctionWebSearch],
tool_search_requested: bool,
tool_search: ToolSearchRouting,
) -> ResponseFunctionWebSearch | OutputToolSearchCall | None:
web_search_call: Final = web_search_calls.get(tool_id)
if web_search_call is not None:
return web_search_call
if tool_search_requested and is_anthropic_tool_search_use(tool_id, tool_name):
return build_tool_search_call(tool_id, tool_arguments, "completed")
return None
execution: Final = tool_search.execution_of(tool_id, tool_name)
if execution is None:
return None
return build_tool_search_call(tool_id, tool_arguments, "completed", execution)
@staticmethod
def _web_search_calls_by_call_id(

View file

@ -106,7 +106,17 @@ if TYPE_CHECKING:
else:
MCPTool = Any
from .streaming_iterator import BaseResponsesAPIStreamingIterator
from .retrieve_stream_query import (
RETRIEVE_INCLUDE_QUERY_KEYS,
InvalidRetrieveStreamQuery,
RetrieveStreamQuery,
parse_retrieve_stream_query,
)
from .streaming_iterator import (
BaseResponsesAPIStreamingIterator,
ResponsesAPIStreamingIterator,
SyncResponsesAPIStreamingIterator,
)
__all__ = (
"acancel_responses",
@ -192,6 +202,7 @@ def mock_responses_api_response(
_STR_MAPPING: Final = TypeAdapter(Mapping[str, object])
_STR_DICT: Final = TypeAdapter(dict[str, object])
_FORWARDED_VALUE: Final = TypeAdapter(object)
@ -2013,6 +2024,53 @@ def delete_responses(
)
def _requested_retrieve_stream_query(
stream: bool | None,
starting_after: int | None,
extra_query: Mapping[str, object] | None,
custom_llm_provider: str | None,
drop_malformed: bool,
) -> RetrieveStreamQuery:
explicit: Final = {
key: value for key, value in (("stream", stream), ("starting_after", starting_after)) if value is not None
}
requested: Final = {**explicit, **(extra_query or {})}
parsed: Final = parse_retrieve_stream_query(
requested.get("stream"), requested.get("starting_after"), drop_malformed=drop_malformed
)
if isinstance(parsed, InvalidRetrieveStreamQuery):
raise litellm.BadRequestError(
message=parsed.message,
model=None,
llm_provider=custom_llm_provider,
body={"message": parsed.message, "type": "invalid_request_error", "param": parsed.param},
)
return parsed
def _supported_retrieve_stream_query(
requested: RetrieveStreamQuery,
responses_api_provider_config: BaseResponsesAPIConfig,
custom_llm_provider: str,
drop_params: object,
) -> RetrieveStreamQuery:
if not requested.stream:
return RetrieveStreamQuery(stream=False, starting_after=None)
if responses_api_provider_config.supports_streamed_retrieval():
return requested
if drops_unsupported_params(drop_params):
return RetrieveStreamQuery(stream=False, starting_after=None)
raise UnsupportedParamsError(
status_code=400,
llm_provider=custom_llm_provider,
message=(
f"{custom_llm_provider} does not support streaming a stored response (stream=true on "
"GET /v1/responses/{response_id}). To drop it, set `litellm.drop_params=True` or for proxy:\n\n"
"`litellm_settings:\n drop_params: true`\n"
),
)
@client
async def aget_responses(
response_id: str,
@ -2024,8 +2082,10 @@ async def aget_responses(
timeout: float | httpx.Timeout | None = None,
# LiteLLM specific params,
custom_llm_provider: str | None = None,
stream: bool | None = None,
starting_after: int | None = None,
**kwargs,
) -> ResponsesAPIResponse:
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
"""
Async: Fetch a response by its ID.
@ -2058,6 +2118,8 @@ async def aget_responses(
extra_query=extra_query,
extra_body=extra_body,
timeout=timeout,
stream=stream,
starting_after=starting_after,
**kwargs,
)
@ -2071,7 +2133,7 @@ async def aget_responses(
response = init_response
# Update the responses_api_response_id with the model_id
if not isinstance(response, ResponsesAPIResponse): # pyright: ignore[reportUnnecessaryIsInstance] # handlers can return non-ResponsesAPIResponse objects at runtime
if not isinstance(response, ResponsesAPIResponse):
return response
return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id(
responses_api_response=response,
@ -2101,8 +2163,14 @@ def get_responses(
timeout: float | httpx.Timeout | None = None,
# LiteLLM specific params,
custom_llm_provider: str | None = None,
stream: bool | None = None,
starting_after: int | None = None,
**kwargs,
) -> ResponsesAPIResponse | Coroutine[object, object, ResponsesAPIResponse]:
) -> (
ResponsesAPIResponse
| SyncResponsesAPIStreamingIterator
| Coroutine[object, object, ResponsesAPIResponse | ResponsesAPIStreamingIterator]
):
"""
Fetch a response by its ID.
@ -2146,6 +2214,20 @@ def get_responses(
if responses_api_provider_config is None:
raise ValueError(f"GET responses is not supported for {custom_llm_provider}")
stream_query: Final = _supported_retrieve_stream_query(
requested=_requested_retrieve_stream_query(
stream=stream,
starting_after=starting_after,
extra_query=extra_query,
custom_llm_provider=custom_llm_provider,
drop_malformed=drops_unsupported_params(litellm_params.drop_params),
),
responses_api_provider_config=responses_api_provider_config,
custom_llm_provider=custom_llm_provider,
drop_params=litellm_params.drop_params,
)
litellm_metadata: Final = _STR_DICT.validate_python(kwargs.get("litellm_metadata") or {})
local_vars.update(kwargs)
# Pre Call logging
@ -2169,7 +2251,13 @@ def get_responses(
litellm_params=litellm_params,
logging_obj=litellm_logging_obj,
extra_headers=extra_headers,
extra_query={
key: value for key, value in (extra_query or {}).items() if key in RETRIEVE_INCLUDE_QUERY_KEYS
},
extra_body=extra_body,
stream=stream_query.stream,
starting_after=stream_query.starting_after,
litellm_metadata=litellm_metadata,
timeout=timeout or request_timeout,
_is_async=_is_async,
client=kwargs.get("client"),
@ -2180,7 +2268,7 @@ def get_responses(
if isinstance(response, ResponsesAPIResponse):
response = ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id(
responses_api_response=response,
litellm_metadata=kwargs.get("litellm_metadata", {}),
litellm_metadata=litellm_metadata,
custom_llm_provider=custom_llm_provider,
)

View file

@ -0,0 +1,83 @@
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Final, Literal
from pydantic import TypeAdapter
RETRIEVE_STREAM_QUERY_KEYS: Final = frozenset({"stream", "starting_after"})
RETRIEVE_INCLUDE_QUERY_KEYS: Final = frozenset({"include", "include[]", "include_obfuscation"})
_QUERY_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_MAX_STARTING_AFTER_DIGITS: Final = len(str(2**63 - 1))
@dataclass(frozen=True, slots=True)
class RetrieveStreamQuery:
stream: bool
starting_after: int | None
@dataclass(frozen=True, slots=True)
class InvalidRetrieveStreamQuery:
param: Literal["stream", "starting_after"]
message: str
_NO_STREAM: Final = RetrieveStreamQuery(stream=False, starting_after=None)
_FULL_REPLAY: Final = RetrieveStreamQuery(stream=True, starting_after=None)
def _parsed_stream(value: object) -> bool | InvalidRetrieveStreamQuery:
match value:
case None:
return False
case bool():
return value
case str() if value.lower() in ("true", "false"):
return value.lower() == "true"
case _:
return InvalidRetrieveStreamQuery("stream", f"Invalid 'stream': expected true or false, got {value!r}")
def _parsed_starting_after(value: object) -> int | None | InvalidRetrieveStreamQuery:
match value:
case None:
return None
case int() if not isinstance(value, bool) and value >= 0:
return value
case str() if value.isascii() and value.isdigit() and len(value) <= _MAX_STARTING_AFTER_DIGITS:
return int(value)
case _:
return InvalidRetrieveStreamQuery(
"starting_after", f"Invalid 'starting_after': expected an integer >= 0, got {value!r}"
)
def parse_retrieve_stream_query(
stream: object, starting_after: object, drop_malformed: bool
) -> RetrieveStreamQuery | InvalidRetrieveStreamQuery:
parsed_stream: Final = _parsed_stream(stream)
if isinstance(parsed_stream, InvalidRetrieveStreamQuery):
return _NO_STREAM if drop_malformed else parsed_stream
if not parsed_stream:
return _NO_STREAM
parsed_starting_after: Final = _parsed_starting_after(starting_after)
if isinstance(parsed_starting_after, InvalidRetrieveStreamQuery):
return _FULL_REPLAY if drop_malformed else parsed_starting_after
return RetrieveStreamQuery(stream=True, starting_after=parsed_starting_after)
def requests_streamed_retrieval(request: Mapping[str, object]) -> bool:
extra_query: Final = request.get("extra_query")
return (
"response_id" in request
and isinstance(extra_query, Mapping)
and _parsed_stream(_QUERY_ADAPTER.validate_python(extra_query).get("stream")) is True
)
def without_retrieve_stream_defaults(
deployment_params: Mapping[str, object], request: Mapping[str, object]
) -> Mapping[str, object]:
if "response_id" not in request:
return deployment_params
return {key: value for key, value in deployment_params.items() if key not in RETRIEVE_STREAM_QUERY_KEYS}

View file

@ -376,6 +376,7 @@ class BaseResponsesAPIStreamingIterator:
_pending_logging_tasks: tuple[asyncio.Task[object], ...] = ()
_started_tool_call: bool = False
_earlier_rounds_spend: BilledRoundsSpend | None = None
_replays_stored_stream: bool = False
def __init__(
self,
@ -574,7 +575,7 @@ class BaseResponsesAPIStreamingIterator:
openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED,
):
_response_obj: Final[object] = getattr(openai_responses_api_chunk, "response", None)
_estimate_wanted: Final[bool] = _chunk_type in (
_estimate_wanted: Final[bool] = not self._replays_stored_stream and _chunk_type in (
openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE,
)
@ -995,6 +996,7 @@ class BaseResponsesAPIStreamingIterator:
generated_text: Final = self._generated_content + self._generated_tool_arguments
if (
not generated_text
or self._replays_stored_stream
or requests_background(self.request_data)
or is_background_response(self._created_response)
or self._completed_response_logged
@ -1110,7 +1112,7 @@ class BaseResponsesAPIStreamingIterator:
"""
created: Final = self._created_response
generated: Final = self._generated_content + self._generated_tool_arguments
if created is None or not generated or self._billed_without_disconnect():
if self._replays_stored_stream or created is None or not generated or self._billed_without_disconnect():
return None
usage: Final = _estimate_usage_safely(
self.model or "",
@ -1162,6 +1164,8 @@ class BaseResponsesAPIStreamingIterator:
def _raise_if_ended_without_terminal_event(self) -> None:
if self.completed_response is not None:
return
if self._replays_stored_stream and not self._yielded_first_chunk:
return
error: Final = self._stream_ended_early_error()
self._handle_failure(error)
if self._output_started:
@ -1209,6 +1213,7 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
custom_llm_provider: str | None = None,
request_data: dict[str, object] | None = None,
call_type: str | None = None,
replays_stored_stream: bool = False,
):
super().__init__(
response,
@ -1220,6 +1225,7 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
request_data,
call_type,
)
self._replays_stored_stream = replays_stored_stream
self.stream_iterator = SSEDecoder().aiter_bytes(response.aiter_bytes())
def __aiter__(self):
@ -1232,6 +1238,9 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
await self._await_pending_logging()
raise
async def aclose(self) -> None:
await self.response.aclose()
async def _next_event(self) -> ResponsesAPIStreamingResponse:
try:
self._check_max_streaming_duration()
@ -1304,6 +1313,7 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
custom_llm_provider: str | None = None,
request_data: dict[str, object] | None = None,
call_type: str | None = None,
replays_stored_stream: bool = False,
):
super().__init__(
response,
@ -1315,11 +1325,15 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
request_data,
call_type,
)
self._replays_stored_stream = replays_stored_stream
self.stream_iterator = SSEDecoder().iter_bytes(response.iter_bytes())
def __iter__(self):
return self
def close(self) -> None:
self.response.close()
def __next__(self):
try:
self._check_max_streaming_duration()

View file

@ -184,6 +184,7 @@ from litellm.llms.openai_like.model_info import (
get_openai_compatible_model_info,
)
from litellm.responses.bridge_check import keep_registered_mode
from litellm.responses.retrieve_stream_query import requests_streamed_retrieval, without_retrieve_stream_defaults
from litellm.router_strategy.auto_router.auto_router_model_naming import (
AUTO_ROUTER_MODEL_PREFIX,
STRATEGY_ROUTER_PARAM_FIELDS,
@ -723,6 +724,29 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj
return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {})
def _search_tool_support(deployment: DeploymentTypedDict) -> bool | None:
from litellm.responses.litellm_completion_transformation.tool_search import chat_config_builds_anthropic_request
configured: Final = _configured_model_info(deployment).get("supports_search_tool")
if isinstance(configured, bool):
return configured
params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(deployment["litellm_params"])
model: Final = params.get("model")
provider: Final = params.get("custom_llm_provider")
if not isinstance(model, str):
return None
try:
resolved_model, resolved_provider = resolve_model_and_provider_for_metadata(
model=model, custom_llm_provider=provider if isinstance(provider, str) else None
)
chat_config: Final = ProviderConfigManager.get_provider_chat_config(
model=resolved_model, provider=LlmProviders(resolved_provider)
)
except (litellm.BadRequestError, ValueError):
return None
return True if chat_config_builds_anthropic_request(chat_config) else None
def _every_deployment_refused(
deployments: Sequence[Mapping[str, object]], refused_deployment_ids: tuple[str, ...]
) -> bool:
@ -4837,7 +4861,7 @@ class Router:
Handles inserting this as either "metadata" or "litellm_metadata" depending on the metadata_variable_name
"""
# 1) copy your defaults and pull out metadata
defaults: Final = self.default_litellm_params.copy()
defaults: Final = dict(without_retrieve_stream_defaults(self.default_litellm_params, kwargs))
metadata_defaults: Final = defaults.pop("metadata", {}) or {}
# 2) add any non-metadata defaults that aren't already in kwargs
@ -5078,12 +5102,13 @@ class Router:
router_timeout=_router_timeout,
router_stream_timeout=_router_stream_timeout,
)
timeout_request: Final = {**kwargs, "stream": True} if requests_streamed_retrieval(kwargs) else kwargs
kwargs.update(
CompletionTimeout.stream_request_kwargs(
passthrough_timeout(kwargs={**kwargs, "stream": False}), passthrough_timeout(kwargs=kwargs)
passthrough_timeout(kwargs={**kwargs, "stream": False}), passthrough_timeout(kwargs=timeout_request)
)
if splits_stream_timeout and kwargs.get("stream")
else {"timeout": passthrough_timeout(kwargs=kwargs)}
if splits_stream_timeout and timeout_request.get("stream")
else {"timeout": passthrough_timeout(kwargs=timeout_request)}
)
elif splits_stream_timeout:
request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(kwargs)
@ -6417,7 +6442,7 @@ class Router:
generic_function_name: Final[str] = getattr(original_generic_function, "__name__", "")
response_kwargs: Final = {
**data,
**without_retrieve_stream_defaults(data, kwargs),
"caching": self.cache_responses,
**kwargs,
"model": model_name,
@ -12234,6 +12259,15 @@ class Router:
for deployment in self._routable_deployments(model_name, team_id)
)
def get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]:
"""
Whether each deployment a request from team_id to model_name can route
to (see _routable_deployments) answers Codex's client `tool_search`:
its model_info.supports_search_tool when that is a bool, else True when
its route builds an Anthropic Messages request, else None (unknown).
"""
return tuple(_search_tool_support(deployment) for deployment in self._routable_deployments(model_name, team_id))
def get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None:
"""
Return the `litellm_params.model` of the first deployment a request

View file

@ -96,6 +96,8 @@ class Feature(str, Enum):
"""litellm.include_cost_in_streaming_usage, where Python adds cost to streamed usage."""
THINKING_SIGNATURE_RETRY = "thinking_signature_retry"
"""Replayed thinking blocks, which Python strips and retries once if Anthropic rejects their signature."""
TOOL_CUSTOM_FIELD = "tool_custom_field"
"""Tools carrying a non-spec custom object, which Python drops or hoists for the provider."""
CLIENT_HEADERS = "client_headers"
"""Proxy-forwarded headers that Python merges into extra_headers."""
DEPLOYMENT_HOOK = "deployment_hook"
@ -176,6 +178,7 @@ MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
Feature.CACHE_CONTROL_INJECTION,
Feature.STREAM_COST_INJECTION,
Feature.THINKING_SIGNATURE_RETRY,
Feature.TOOL_CUSTOM_FIELD,
}
RESPONSES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {

View file

@ -43,6 +43,11 @@ def thinking_signature_retry(request: NativeCall) -> bool:
return any(_replays_thinking(message) for message in optional_sequence(request.resolved.get("messages")) or ())
def tool_custom_field(request: NativeCall) -> bool:
tools: Final = optional_sequence(request.resolved.get("tools")) or ()
return any("custom" in (optional_mapping(tool) or {}) for tool in tools)
def stream_cost_injection(request: NativeCall) -> bool:
"""Python adds response cost to streaming usage when the global setting is truthy."""
return streaming(request) and bool(litellm.include_cost_in_streaming_usage)

View file

@ -63,6 +63,8 @@ def _assistant_tool_call(tool_call: ChatCompletionMessageToolCall) -> ChatComple
def _assistant_message(
message: Message, tool_calls: tuple[ChatCompletionMessageToolCall, ...], replays_provider_specific_fields: bool
) -> ChatCompletionAssistantMessage:
from litellm.completion_extras.litellm_responses_transformation.history import retained_replay_fields
return cast( # cast-ok: dict literal with the optional message fields spread in only when set
"ChatCompletionAssistantMessage",
{
@ -78,7 +80,7 @@ def _assistant_message(
"provider_specific_fields",
getattr(message, "provider_specific_fields", None)
if replays_provider_specific_fields
else None,
else retained_replay_fields(getattr(message, "provider_specific_fields", None)),
),
)
if value is not None

View file

@ -4424,6 +4424,7 @@ class LlmProviders(str, Enum):
JINA_AI = "jina_ai"
XAI = "xai"
ZAI = "zai"
BIGMODEL = "bigmodel"
CUSTOM_OPENAI = "custom_openai"
TEXT_COMPLETION_OPENAI = "text-completion-openai"
COHERE = "cohere"

View file

@ -2032,16 +2032,15 @@ def client(original_function):
# MODEL CALL
result = original_function(*args, **kwargs)
end_time = datetime.datetime.now()
if _is_streaming_request(
kwargs=kwargs,
call_type=call_type,
):
if _is_streaming_request(kwargs=kwargs, call_type=call_type) or _is_converted_stream_result(result):
if "complete_response" in kwargs and kwargs["complete_response"] is True:
chunks = []
for idx, chunk in enumerate(result):
chunks.append(chunk)
return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages", None))
else:
logging_obj.stream = True
logging_obj.model_call_details["stream"] = True
# RETURN RESULT
update_response_metadata: _ResponseMetadataUpdater = litellm_utils.update_response_metadata
update_response_metadata(
@ -2578,6 +2577,8 @@ _NEVER_STREAMING_CALL_TYPES: Final = frozenset(
}
)
_RESULT_DECIDES_STREAMING_CALL_TYPES: Final = frozenset({"get_responses", "aget_responses"})
def _is_streaming_request(
kwargs: dict[str, object],
@ -2589,7 +2590,7 @@ def _is_streaming_request(
- if "stream=True" in kwargs (litellm chat completion, litellm text completion, litellm messages)
- if call_type is generate_content_stream or agenerate_content_stream (litellm google genai)
"""
if call_type in _NEVER_STREAMING_CALL_TYPES:
if call_type in _NEVER_STREAMING_CALL_TYPES or call_type in _RESULT_DECIDES_STREAMING_CALL_TYPES:
return False
if "stream" in kwargs and kwargs["stream"] is True:
return True
@ -5674,6 +5675,7 @@ def add_provider_specific_params_to_optional_params(
"openai",
"azure",
"text-completion-openai",
"bigmodel",
*litellm.openai_compatible_providers,
] or JSONProviderRegistry.exists(custom_llm_provider)
if calls_through_openai_sdk:
@ -9183,6 +9185,7 @@ class ProviderConfigManager:
LlmProviders.DATABRICKS: (lambda: litellm.DatabricksConfig(), False),
LlmProviders.XAI: (lambda: litellm.XAIChatConfig(), False),
LlmProviders.ZAI: (lambda: litellm.ZAIChatConfig(), False),
LlmProviders.BIGMODEL: (ProviderConfigManager._get_bigmodel_chat_config, False),
LlmProviders.OPENCODE_GO: (ProviderConfigManager.get_opencode_go_config, True),
LlmProviders.LAMBDA_AI: (lambda: litellm.LambdaAIChatConfig(), False),
LlmProviders.INCEPTION: (lambda: litellm.InceptionChatConfig(), False),
@ -9375,6 +9378,12 @@ class ProviderConfigManager:
return get_bedrock_chat_config(model=model)
@staticmethod
def _get_bigmodel_chat_config() -> BaseConfig:
from litellm.llms.bigmodel.chat.transformation import BigModelChatConfig
return BigModelChatConfig()
@staticmethod
def get_opencode_go_config(model: str) -> BaseConfig:
from litellm.llms.opencode.chat.transformation import go_chat_config
@ -9696,6 +9705,10 @@ class ProviderConfigManager:
model: str,
provider: LlmProviders,
) -> BaseAnthropicMessagesConfig | None:
if provider == LlmProviders.BIGMODEL:
from litellm.llms.bigmodel.messages.transformation import BigModelMessagesConfig
return BigModelMessagesConfig()
if provider == LlmProviders.OPENCODE_GO:
from litellm.llms.opencode.common_utils import go_endpoint
from litellm.llms.opencode.messages.transformation import OpenCodeGoMessagesConfig
@ -9927,6 +9940,10 @@ class ProviderConfigManager:
if provider is None:
return None
if provider == LlmProviders.BIGMODEL:
from litellm.llms.bigmodel.responses.transformation import BigModelResponsesAPIConfig
return BigModelResponsesAPIConfig()
if provider == LlmProviders.OPENCODE_GO:
from litellm.llms.opencode.common_utils import go_endpoint
from litellm.llms.opencode.responses.transformation import OpenCodeGoResponsesConfig

View file

@ -78995,5 +78995,81 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"bigmodel/glm-5.3": {
"input_cost_per_token": 1.1428571428571428e-06,
"output_cost_per_token": 4e-06,
"cache_read_input_token_cost": 2.857142857142857e-07,
"litellm_provider": "bigmodel",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"mode": "chat",
"source": "https://docs.bigmodel.cn/cn/guide/start/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"thinking_always_on": true,
"metadata": {
"comment": "Official API list price checked 2026-10-27: input 8, output 28, cached input 2 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded."
},
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses",
"/v1/messages"
],
"supports_native_streaming": true
},
"bigmodel/glm-5.3-flash": {
"input_cost_per_token": 1.142857142857143e-07,
"output_cost_per_token": 4e-07,
"cache_read_input_token_cost": 3.2857142857142855e-08,
"litellm_provider": "bigmodel",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"mode": "chat",
"source": "https://docs.bigmodel.cn/cn/guide/start/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"thinking_always_on": true,
"supports_vision": true,
"supports_video_input": true,
"metadata": {
"comment": "Official API list price checked 2026-10-27: input 0.8, output 2.8, cached input 0.23 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded."
},
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses",
"/v1/messages"
],
"supports_native_streaming": true
},
"bigmodel/glm-5.3-flashx": {
"input_cost_per_token": 2.857142857142857e-07,
"output_cost_per_token": 1e-06,
"cache_read_input_token_cost": 8.142857142857142e-08,
"litellm_provider": "bigmodel",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"mode": "chat",
"source": "https://docs.bigmodel.cn/cn/guide/start/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"thinking_always_on": true,
"supports_vision": true,
"supports_video_input": true,
"metadata": {
"comment": "Official API list price checked 2026-10-27: input 2, output 7, cached input 0.57 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded."
},
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses",
"/v1/messages"
],
"supports_native_streaming": true
}
}

View file

@ -5326,6 +5326,32 @@
"a2a": false,
"generateContent": false
}
},
"bigmodel": {
"display_name": "BigModel (Zhipu AI) (`bigmodel`)",
"url": "https://docs.bigmodel.cn/cn/guide/develop/responses/introduction",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
}
},
"endpoints": {

View file

@ -107,6 +107,7 @@ ignored_function_names = [
"_deployment_num_retries", # Tested through the deployment num_retries mid-stream budget test in test_router.py
"_request_fallback_list", # Tested through every mid-stream retry test in test_router.py
"_request_model_group", # Tested through test_anthropic_messages_retry_budget_precedence_direct_call
"_search_tool_support", # Tested through the test_get_search_tool_support_* tests
"_mid_stream_retry_trigger", # Tested through the retry policy mid-stream budget test in test_router.py
"_anthropic_messages_group_retry_policy", # Tested through the retry budget precedence test in test_router.py
"_anthropic_messages_resolved_retry_policy", # Tested through the malformed retry policy tests in test_router.py

View file

@ -102,6 +102,7 @@
"get_routable_upstream_model",
"get_router_deployment_budget_limiter",
"get_routing_group",
"get_search_tool_support",
"get_settings",
"get_valid_args",
"guardrail_list",

View file

@ -155,3 +155,4 @@
- {id: llm.responses.anthropic.basic.stream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /v1/responses streams Responses events"}
- {id: llm.responses.anthropic.basic.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /v1/responses: cost header and spend row agree"}
- {id: llm.responses.anthropic.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic function_call_output round trip over /v1/responses"}
- {id: llm.responses.anthropic.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: tool_search, streaming: nonstream, assertions: [works], source: "llm_translation/test_responses_codex_tool_search_e2e.py", rationale: "Codex client tool_search over /v1/responses finds and loads a deferred tool on Claude"}

View file

@ -0,0 +1,205 @@
"""Live e2e: Codex's client tool_search over /v1/responses on Claude.
Codex holds MCP tools back behind its client-executed `tool_search` tool when the model catalog says
`supports_search_tool`. Over the completion bridge the proxy offers that search to Claude as a plain function,
hands Claude's call back as a `tool_search_call` Codex runs itself, and loads the tools Codex returns in
`tool_search_output` so Claude can call them on the next turn
"""
from __future__ import annotations
from typing import Final
import pytest
from e2e_config import unique_marker
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from lifecycle import ResourceManager
from models import LiteLLMParamsBody
from openai.types.responses import (
FunctionToolParam,
Response,
ResponseFunctionToolCall,
ResponseInputParam,
ResponseToolSearchCall,
ToolParam,
ToolSearchToolParam,
)
from proxy_client import ProxyClient
from pydantic import BaseModel, ConfigDict
from sdk_clients import NO_PROXY_CACHE, SdkClients
pytestmark = pytest.mark.e2e
ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-5-5"
DEFERRED_TOOL_COUNT: Final = 50
CALENDAR_TOOL_NAME: Final = "mcp__codex_apps__calendar_create_event"
CODEX_TOOL_SEARCH: Final[ToolSearchToolParam] = {
"type": "tool_search",
"execution": "client",
"description": (
"# Tool discovery\n\nSearches over deferred tool metadata with BM25 and exposes matching tools for the next "
"model call. Some of the tools may not have been provided to you upfront, and you should use this tool "
"(`tool_search`) to search for the required tools."
),
"parameters": {
"type": "object",
"properties": {
"query": {"type": "string", "description": "Search query for deferred tools."},
"limit": {"type": "number", "description": "Maximum number of tools to return. Defaults to 8."},
},
"required": ["query"],
"additionalProperties": False,
},
}
SHELL_TOOL: Final[FunctionToolParam] = {
"type": "function",
"name": "shell",
"description": "Runs a shell command.",
"parameters": {"type": "object", "properties": {"cmd": {"type": "string"}}, "required": ["cmd"]},
"strict": False,
}
CALENDAR_TOOL: Final[FunctionToolParam] = {
"type": "function",
"name": CALENDAR_TOOL_NAME,
"description": "Create a calendar event.",
"parameters": {
"type": "object",
"properties": {"title": {"type": "string"}},
"required": ["title"],
"additionalProperties": False,
},
"strict": False,
"defer_loading": True,
}
PROMPT: Final = "Create a calendar event titled Standup. Find the calendar tool with tool_search first, then call it."
class _CalendarArguments(BaseModel):
model_config = ConfigDict(frozen=True)
title: str
def _filler_tool(index: int) -> FunctionToolParam:
return {
"type": "function",
"name": f"mcp__codex_apps__records_{index:02d}",
"description": f"List, inspect, update and archive records of family {index} in the connected workspace.",
"parameters": {
"type": "object",
"properties": {
"title": {"type": "string", "description": "Title of the record to act on"},
"notes": {"type": "string", "description": "Free-form notes and context for the operation"},
"tags": {"type": "array", "items": {"type": "string"}},
"priority": {"type": "integer", "minimum": 0, "maximum": 5},
},
"required": ["title"],
"additionalProperties": False,
},
"strict": False,
"defer_loading": True,
}
DEFERRED_TOOLS: Final[tuple[FunctionToolParam, ...]] = (
*(_filler_tool(index) for index in range(DEFERRED_TOOL_COUNT - 1)),
CALENDAR_TOOL,
)
def _eager(tool: FunctionToolParam) -> ToolParam:
return FunctionToolParam(
type="function",
name=tool["name"],
description=tool.get("description"),
parameters=tool.get("parameters"),
strict=False,
)
def _input_tokens(response: Response) -> int:
assert response.usage is not None, f"response carried no usage: {response.id}"
return response.usage.input_tokens
def _search_call(response: Response) -> ResponseToolSearchCall:
calls: Final = tuple(item for item in response.output if isinstance(item, ResponseToolSearchCall))
assert len(calls) == 1, f"expected one tool_search_call, got {response.output!r}"
return calls[0]
class TestResponsesCodexToolSearchOverAnthropic:
@pytest.mark.covers("llm.responses.anthropic.tool_search.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.ANTHROPIC,),
models=(ANTHROPIC_BACKEND,),
capabilities=(Capability.TOOL_SEARCH, Capability.FUNCTION_CALLING),
mode=Mode.NONSTREAM,
)
)
def test_claude_finds_a_deferred_tool_through_codex_tool_search_and_calls_it(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model: Final = f"e2e-codex-tool-search-{unique_marker()}"
model_id: Final = proxy.create_model(
model, LiteLLMParamsBody(model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY")
)
resources.defer(lambda: proxy.delete_model(model_id))
client: Final = sdk.openai(resources.key())
searchable_tools: Final[list[ToolParam]] = [SHELL_TOOL, CODEX_TOOL_SEARCH]
eager: Final = client.responses.create(
model=model,
input=PROMPT,
tools=[SHELL_TOOL, *(_eager(tool) for tool in DEFERRED_TOOLS)],
extra_body=NO_PROXY_CACHE,
)
first: Final = client.responses.create(
model=model, input=PROMPT, tools=searchable_tools, extra_body=NO_PROXY_CACHE
)
search: Final = _search_call(first)
assert search.execution == "client", f"tool_search ran on the wrong side: {search!r}"
assert search.call_id, f"tool_search_call had no call_id: {search!r}"
assert not any(isinstance(item, ResponseFunctionToolCall) for item in first.output), first.output
assert _input_tokens(first) * 4 < _input_tokens(eager), (
f"deferred first turn used {_input_tokens(first)} input tokens against {_input_tokens(eager)} eager"
)
follow_up: Final[ResponseInputParam] = [
{"role": "user", "content": PROMPT},
{
"type": "tool_search_call",
"call_id": search.call_id,
"execution": "client",
"arguments": search.arguments,
},
{
"type": "tool_search_output",
"call_id": search.call_id,
"status": "completed",
"execution": "client",
"tools": [CALENDAR_TOOL],
},
]
second: Final = client.responses.create(
model=model, input=follow_up, tools=searchable_tools, extra_body=NO_PROXY_CACHE
)
calendar_call: Final = next(
(
item
for item in second.output
if isinstance(item, ResponseFunctionToolCall) and item.name == CALENDAR_TOOL_NAME
),
None,
)
assert calendar_call is not None, f"Claude did not call the loaded tool: {second.output!r}"
assert "standup" in _CalendarArguments.model_validate_json(calendar_call.arguments).title.lower()

View file

@ -37,3 +37,14 @@ together with its dedicated job, and list a new dedicated GitHub workflow with t
`DEDICATED_INTEGRATION_WORKFLOWS` in `.github/scripts/assert_ci_coverage.py`. The `ci-coverage` check
fails a file here that a GitHub workflow runs unless it is in `GITHUB_FILES` and that workflow owns its
directory (#46780)
## Fixtures that boot a proxy
Scope them to the module or class, or pin a session one to a single worker with an `xdist_group` mark
as `_llm_gateway` in `mcp/test_mcp_llm_endpoints.py` does, and request them as arguments, never through
`request.getfixturevalue`. The harness puts tests that request the same set of module or class fixtures
on one xdist worker, so the proxy they share boots there once and stops when that worker moves to another
file. A test that requests one more of those fixtures than its neighbors gets its own group and can boot
the proxy again on another worker, so keep that set the same across the tests that share a proxy. An
unpinned session fixture boots a copy on every worker that runs one of its tests and keeps them all until
the node ends, and those copies ran CircleCI integration-extensions nodes out of memory

View file

@ -0,0 +1,214 @@
import json
import uuid
from collections.abc import Callable, Sequence
from itertools import chain
from typing import Final
from integration._support.client import list_value, object_value
from integration._support.wire import Reply, Request, Wire
from pydantic import JsonValue
MODEL: Final = "gpt-6.1-sol"
PLAN: Final = "I am starting by choosing which city to check."
PICK: Final = "My pick is Tokyo, so I will check its weather."
PATCH_NOTE: Final = "I will create hello.txt containing hi."
ANSWER: Final = "It is raining in Tokyo."
WEATHER_TOOL: Final[dict[str, JsonValue]] = {
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the weather for a city.",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
},
}
PATCH_TOOL: Final[dict[str, JsonValue]] = {
"type": "custom",
"custom": {"name": "apply_patch", "description": "Apply a V4A patch to the workspace"},
}
def _message(item_id: str, text: str, phase: str) -> dict[str, JsonValue]:
return {
"type": "message",
"id": item_id,
"status": "completed",
"role": "assistant",
"phase": phase,
"content": [{"type": "output_text", "text": text, "annotations": []}],
}
def _reasoning(item_id: str, summary: str) -> dict[str, JsonValue]:
return {
"type": "reasoning",
"id": item_id,
"summary": [{"type": "summary_text", "text": summary}],
"encrypted_content": f"enc_{item_id}",
}
_WEATHER_CALL: Final[dict[str, JsonValue]] = {
"type": "function_call",
"id": "fc_weather",
"call_id": "call_weather",
"name": "get_weather",
"arguments": '{"city":"Tokyo"}',
"status": "completed",
}
WEATHER_TURN: Final[tuple[dict[str, JsonValue], ...]] = (
_message("msg_plan", PLAN, "commentary"),
_reasoning("rs_pick", "Tokyo is often rainy this time of year."),
_message("msg_pick", PICK, "commentary"),
_WEATHER_CALL,
)
TWO_THOUGHTS_TURN: Final[tuple[dict[str, JsonValue], ...]] = (
_reasoning("rs_scan", "Paris and Tokyo are the two candidates."),
_reasoning("rs_pick", "Tokyo is often rainy this time of year."),
_message("msg_pick", PICK, "commentary"),
_WEATHER_CALL,
)
CALL_ONLY_TURN: Final[tuple[dict[str, JsonValue], ...]] = (_WEATHER_CALL,)
PATCH_TURN: Final[tuple[dict[str, JsonValue], ...]] = (
_message("msg_patch", PATCH_NOTE, "commentary"),
{
"type": "custom_tool_call",
"id": "ctc_patch",
"call_id": "call_patch",
"name": "apply_patch",
"input": "*** Begin Patch\n*** Add File: hello.txt\n+hi\n*** End Patch",
"status": "completed",
},
)
WEATHER_REPLAY: Final = (
("message", "assistant", "commentary", PLAN),
("reasoning", "rs_pick"),
("message", "assistant", "commentary", PICK),
("function_call", "call_weather", "get_weather"),
("function_call_output", "call_weather"),
)
TWO_THOUGHTS_REPLAY: Final = (
("reasoning", "rs_scan"),
("reasoning", "rs_pick"),
("message", "assistant", "commentary", PICK),
("function_call", "call_weather", "get_weather"),
("function_call_output", "call_weather"),
)
def _response(identity: str, output: Sequence[dict[str, JsonValue]]) -> dict[str, JsonValue]:
return {
"id": f"resp_{identity}",
"object": "response",
"created_at": 1789788253,
"status": "completed",
"model": MODEL,
"output": list(output),
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
}
def _event(payload: dict[str, JsonValue]) -> bytes:
return f"event: {payload['type']}\ndata: {json.dumps(payload)}\n\n".encode()
def _opened(item: dict[str, JsonValue]) -> dict[str, JsonValue]:
match item["type"]:
case "message":
return {**item, "status": "in_progress", "content": []}
case "function_call":
return {**item, "status": "in_progress", "arguments": ""}
case _:
return item
def _deltas(index: int, item: dict[str, JsonValue]) -> tuple[dict[str, JsonValue], ...]:
match item["type"]:
case "message":
text: Final = object_value(list_value(item["content"])[0])["text"]
return (
{
"type": "response.output_text.delta",
"item_id": item["id"],
"output_index": index,
"content_index": 0,
"delta": text,
},
)
case "function_call":
return (
{
"type": "response.function_call_arguments.delta",
"item_id": item["id"],
"output_index": index,
"delta": item["arguments"],
},
)
case _:
return ()
def _item_events(index: int, item: dict[str, JsonValue]) -> tuple[bytes, ...]:
return (
_event({"type": "response.output_item.added", "output_index": index, "item": _opened(item)}),
*(_event(delta) for delta in _deltas(index, item)),
_event({"type": "response.output_item.done", "output_index": index, "item": item}),
)
def _stream(identity: str, output: Sequence[dict[str, JsonValue]]) -> tuple[bytes, ...]:
response: Final = _response(identity, output)
return (
_event({"type": "response.created", "response": {**response, "status": "in_progress", "output": []}}),
*chain.from_iterable(_item_events(index, item) for index, item in enumerate(output)),
_event({"type": "response.completed", "response": response}),
)
def _answered(body: dict[str, JsonValue]) -> bool:
return any(
object_value(item).get("type") in {"function_call_output", "custom_tool_call_output"}
for item in list_value(body["input"])
)
def responses_upstream(turn_one: Sequence[dict[str, JsonValue]]) -> Callable[[Request], Reply]:
def respond(request: Request) -> Reply:
if request.method == "GET":
return Reply(body=b'{"object":"list","data":[]}')
assert request.target == "/responses", request.target
body: Final = object_value(json.loads(request.body))
identity: Final = uuid.uuid4().hex
output: Final = (_message("msg_answer", ANSWER, "final_answer"),) if _answered(body) else turn_one
if body.get("stream"):
return Reply(content_type="text/event-stream", chunks=_stream(identity, output))
return Reply(body=json.dumps(_response(identity, output)).encode())
return respond
def _text(content: JsonValue) -> JsonValue:
if isinstance(content, str):
return content
return "".join(str(object_value(part).get("text")) for part in list_value(content))
def _shape(item: JsonValue) -> tuple[JsonValue, ...]:
fields: Final = object_value(item)
match fields.get("type"):
case "message":
return ("message", fields.get("role"), fields.get("phase"), _text(fields.get("content")))
case "reasoning":
return ("reasoning", fields.get("id"))
case "function_call" | "custom_tool_call" as kind:
return (kind, fields.get("call_id"), fields.get("name"))
case kind:
return (kind, fields.get("call_id"))
def posted_inputs(wire: Wire) -> tuple[tuple[tuple[JsonValue, ...], ...], ...]:
return tuple(
tuple(_shape(item) for item in list_value(object_value(json.loads(request.body))["input"]))
for request in wire.drain()
if request.method == "POST"
)

View file

@ -34,7 +34,7 @@ def langfuse_vars(audit_sinks: SpanSinks) -> dict[str, JsonValue]:
return langfuse_callback_variables(audit_sinks)
@pytest.fixture(scope="session")
@pytest.fixture(scope="module")
def capped(tmp_path_factory: pytest.TempPathFactory) -> Iterator[CapRig]:
"""A two-worker proxy capped at three series per metric, with three keys already holding a series each."""
with series_cap_rig(

View file

@ -305,12 +305,12 @@ class RigFactory:
yield Rig(owned.gateway, owned, model, self.provider, self.sink)
@pytest.fixture(scope="session")
@pytest.fixture(scope="module")
def rig(provider: Wire, collector: Collector, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
yield from RigFactory(provider, collector, tmp_path_factory.mktemp("otel"), {}, 2).start()
@pytest.fixture(scope="session")
@pytest.fixture(scope="module")
def generating_rig(provider: Wire, collector: Collector, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
factory: Final = RigFactory(
provider, collector, tmp_path_factory.mktemp("otel-generate"), {"missing_session_id": "generate"}, 2
@ -318,7 +318,7 @@ def generating_rig(provider: Wire, collector: Collector, tmp_path_factory: pytes
yield from factory.start()
@pytest.fixture(scope="session")
@pytest.fixture(scope="module")
def two_worker_rig(provider: Wire, collector: Collector, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
yield from RigFactory(provider, collector, tmp_path_factory.mktemp("otel-workers"), {}, 2).start()

View file

@ -424,7 +424,7 @@ class RigFactory:
yield Rig(owned.gateway, owned, model, self.provider, self.sink, self.tenant_sink)
@pytest.fixture(scope="session")
@pytest.fixture(scope="module")
def rig(
provider: Wire, operator_sink: Collector, tenant_sink: Collector, tmp_path_factory: pytest.TempPathFactory
) -> Iterator[Rig]:
@ -434,7 +434,7 @@ def rig(
yield from factory.start()
@pytest.fixture(scope="session")
@pytest.fixture(scope="module")
def v2_rig(
provider: Wire, operator_sink: Collector, tenant_sink: Collector, tmp_path_factory: pytest.TempPathFactory
) -> Iterator[Rig]:

View file

@ -0,0 +1,233 @@
import json
import uuid
from collections.abc import Sequence
from functools import reduce
from itertools import chain
from typing import Final
import httpx
import pytest
from integration._support.client import Gateway, list_value, object_value
from integration._support.responses_replay import (
CALL_ONLY_TURN,
MODEL,
PATCH_NOTE,
PATCH_TOOL,
PATCH_TURN,
PICK,
PLAN,
TWO_THOUGHTS_REPLAY,
TWO_THOUGHTS_TURN,
WEATHER_REPLAY,
WEATHER_TOOL,
WEATHER_TURN,
posted_inputs,
responses_upstream,
)
from integration._support.wire import wire_server
from pydantic import JsonValue
_API_KEY: Final = "synthetic-openai-key"
def _chunks(response: httpx.Response) -> list[dict[str, JsonValue]]:
return [
object_value(json.loads(line[6:]))
for line in response.iter_lines()
if line.startswith("data: ") and line != "data: [DONE]"
]
def _tool_call(index: JsonValue, calls: Sequence[dict[str, JsonValue]]) -> dict[str, JsonValue]:
parts: Final = [call for call in calls if call["index"] == index]
functions: Final = [object_value(part["function"]) for part in parts]
return {
"id": next(part["id"] for part in parts if part.get("id")),
"type": "function",
"function": {
"name": next(function["name"] for function in functions if function.get("name")),
"arguments": "".join(str(function.get("arguments") or "") for function in functions),
},
}
def _deltas(chunks: Sequence[dict[str, JsonValue]]) -> list[dict[str, JsonValue]]:
return [
object_value(object_value(choice)["delta"])
for choice in chain.from_iterable(list_value(chunk["choices"]) for chunk in chunks)
]
def _assembled(chunks: Sequence[dict[str, JsonValue]]) -> dict[str, JsonValue]:
deltas: Final = _deltas(chunks)
calls: Final = [object_value(call) for call in chain.from_iterable(d.get("tool_calls") or [] for d in deltas)]
fields: Final = reduce(
lambda merged, delta: {**merged, **object_value(delta.get("provider_specific_fields") or {})}, deltas, {}
)
return {
"role": "assistant",
"content": "".join(str(delta["content"]) for delta in deltas if delta.get("content")) or None,
"tool_calls": [_tool_call(index, calls) for index in sorted({int(str(call["index"])) for call in calls})],
"reasoning_items": list(chain.from_iterable(list_value(d.get("reasoning_items") or []) for d in deltas)),
"provider_specific_fields": fields,
}
def _first_turn(
gateway: Gateway, model: str, prompt: str, tools: Sequence[dict[str, JsonValue]], *, stream: bool
) -> dict[str, JsonValue]:
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [{"role": "user", "content": prompt}],
"tools": list(tools),
"stream": stream,
"cache": {"no-cache": True},
},
)
assert response.status_code == 200, response.text
if stream:
return _assembled(_chunks(response))
return object_value(object_value(list_value(response.json()["choices"])[0])["message"])
def _next_turn(
gateway: Gateway,
model: str,
prompt: str,
tools: Sequence[dict[str, JsonValue]],
message: dict[str, JsonValue],
**extra: JsonValue,
) -> httpx.Response:
call_id: Final = object_value(list_value(message["tool_calls"])[0])["id"]
return gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [
{"role": "user", "content": prompt},
message,
{"role": "tool", "tool_call_id": call_id, "content": "Rain, 18C"},
],
"tools": list(tools),
"cache": {"no-cache": True},
**extra,
},
)
def _with_phase(message: dict[str, JsonValue], phase: str) -> dict[str, JsonValue]:
fields: Final = object_value(message.get("provider_specific_fields") or {})
return {**message, "provider_specific_fields": {**fields, "phase": phase}}
def _assistant_messages(items: Sequence[tuple[JsonValue, ...]]) -> list[tuple[JsonValue, ...]]:
return [item for item in items if item[:2] == ("message", "assistant")]
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
def test_chat_next_turn_replays_responses_items_in_the_order_openai_produced(gateway: Gateway, stream: bool) -> None:
prompt: Final = f"Pick a city and check whether it is raining there {uuid.uuid4().hex}"
with wire_server(responses_upstream(WEATHER_TURN)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"openai/responses/{MODEL}", api_base=wire.url, api_key=_API_KEY)
message: Final = _first_turn(gateway, model, prompt, [WEATHER_TOOL], stream=stream)
response: Final = _next_turn(gateway, model, prompt, [WEATHER_TOOL], message)
inputs: Final = posted_inputs(wire)
assert response.status_code == 200, response.text
assert message["content"] == PLAN + PICK, message
assert object_value(message["provider_specific_fields"])["phase"] == "commentary", message
assert inputs[1] == (("message", "user", None, prompt), *WEATHER_REPLAY), inputs
def test_chat_next_turn_sends_the_phase_the_app_set_on_the_assistant_message(gateway: Gateway) -> None:
prompt: Final = f"Pick a city and check whether it is raining there {uuid.uuid4().hex}"
with wire_server(responses_upstream(WEATHER_TURN)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"openai/responses/{MODEL}", api_base=wire.url, api_key=_API_KEY)
message: Final = _first_turn(gateway, model, prompt, [WEATHER_TOOL], stream=False)
response: Final = _next_turn(gateway, model, prompt, [WEATHER_TOOL], _with_phase(message, "final_answer"))
inputs: Final = posted_inputs(wire)
assert response.status_code == 200, response.text
assert [item[2] for item in _assistant_messages(inputs[1])] == ["final_answer"], inputs
def test_chat_next_turn_rejects_an_undefined_phase_unless_drop_params_is_set(gateway: Gateway) -> None:
prompt: Final = f"Pick a city and check whether it is raining there {uuid.uuid4().hex}"
with wire_server(responses_upstream(WEATHER_TURN)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"openai/responses/{MODEL}", api_base=wire.url, api_key=_API_KEY)
message: Final = _with_phase(_first_turn(gateway, model, prompt, [WEATHER_TOOL], stream=False), "analysis")
rejected: Final = _next_turn(gateway, model, prompt, [WEATHER_TOOL], message)
dropped: Final = _next_turn(gateway, model, prompt, [WEATHER_TOOL], message, drop_params=True)
inputs: Final = posted_inputs(wire)
assert rejected.status_code == 400, rejected.text
assert "'analysis'" in rejected.text, rejected.text
assert dropped.status_code == 200, dropped.text
assert len(inputs) == 2, inputs
assert [item[2] for item in _assistant_messages(inputs[1])] == [None], inputs
def test_chat_next_turn_sends_the_text_the_app_edited_instead_of_the_recorded_items(gateway: Gateway) -> None:
prompt: Final = f"Pick a city and check whether it is raining there {uuid.uuid4().hex}"
edited_text: Final = "I picked Tokyo."
with wire_server(responses_upstream(WEATHER_TURN)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"openai/responses/{MODEL}", api_base=wire.url, api_key=_API_KEY)
message: Final = _first_turn(gateway, model, prompt, [WEATHER_TOOL], stream=False)
response: Final = _next_turn(gateway, model, prompt, [WEATHER_TOOL], {**message, "content": edited_text})
inputs: Final = posted_inputs(wire)
assert response.status_code == 200, response.text
assert _assistant_messages(inputs[1]) == [("message", "assistant", "commentary", edited_text)], inputs
def test_chat_next_turn_replays_a_custom_tool_call_as_a_custom_tool_call(gateway: Gateway) -> None:
prompt: Final = f"Create hello.txt containing hi {uuid.uuid4().hex}"
with wire_server(responses_upstream(PATCH_TURN)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"openai/responses/{MODEL}", api_base=wire.url, api_key=_API_KEY)
message: Final = _first_turn(gateway, model, prompt, [PATCH_TOOL], stream=False)
response: Final = _next_turn(gateway, model, prompt, [PATCH_TOOL], message)
inputs: Final = posted_inputs(wire)
assert response.status_code == 200, response.text
assert inputs[1] == (
("message", "user", None, prompt),
("message", "assistant", "commentary", PATCH_NOTE),
("custom_tool_call", "call_patch", "apply_patch"),
("custom_tool_call_output", "call_patch"),
), inputs
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
def test_chat_reply_keeps_every_reasoning_item_and_the_next_turn_replays_each_one(
gateway: Gateway, stream: bool
) -> None:
prompt: Final = f"Weigh two cities, then check whether it is raining in your pick {uuid.uuid4().hex}"
with wire_server(responses_upstream(TWO_THOUGHTS_TURN)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"openai/responses/{MODEL}", api_base=wire.url, api_key=_API_KEY)
message: Final = _first_turn(gateway, model, prompt, [WEATHER_TOOL], stream=stream)
response: Final = _next_turn(gateway, model, prompt, [WEATHER_TOOL], message)
inputs: Final = posted_inputs(wire)
assert response.status_code == 200, response.text
assert [object_value(item)["id"] for item in list_value(message["reasoning_items"])] == ["rs_scan", "rs_pick"]
assert inputs[1] == (("message", "user", None, prompt), *TWO_THOUGHTS_REPLAY), inputs
def test_chat_stream_of_a_tool_call_only_turn_carries_null_content_like_the_json_reply(gateway: Gateway) -> None:
prompt: Final = f"Check whether it is raining in Tokyo {uuid.uuid4().hex}"
with wire_server(responses_upstream(CALL_ONLY_TURN)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"openai/responses/{MODEL}", api_base=wire.url, api_key=_API_KEY)
message: Final = _first_turn(gateway, model, prompt, [WEATHER_TOOL], stream=False)
streamed: Final = gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [{"role": "user", "content": prompt}],
"tools": [WEATHER_TOOL],
"stream": True,
"cache": {"no-cache": True},
},
)
assert streamed.status_code == 200, streamed.text
deltas: Final = _deltas(_chunks(streamed))
assert message["content"] is None, message
assert [delta["content"] for delta in deltas if delta.get("content") is not None] == [], deltas

View file

@ -1,6 +1,7 @@
import json
import uuid
from typing import Final
from unittest.mock import ANY
import httpx
import pytest
@ -288,6 +289,7 @@ def test_chat_over_responses_deployment_merges_message_and_function_call(gateway
"index": 0,
}
],
"provider_specific_fields": {"openai_responses_replay": ANY},
},
}
], response.text
@ -397,6 +399,7 @@ def test_chat_over_responses_deployment_keeps_reasoning_with_merged_tool_call(ga
"index": 0,
}
],
"provider_specific_fields": {"openai_responses_replay": ANY},
},
}
], response.text
@ -480,6 +483,7 @@ def test_chat_over_responses_deployment_returns_tool_call_only_reply_as_one_choi
"index": 0,
}
],
"provider_specific_fields": {"openai_responses_replay": ANY},
},
}
], response.text
@ -570,6 +574,7 @@ def test_chat_over_responses_deployment_merges_function_call_followed_by_message
"index": 0,
}
],
"provider_specific_fields": {"openai_responses_replay": ANY},
},
}
], response.text
@ -671,6 +676,7 @@ def test_chat_over_responses_deployment_folds_two_messages_and_function_call_int
"index": 0,
}
],
"provider_specific_fields": {"openai_responses_replay": ANY},
},
}
], response.text
@ -729,6 +735,7 @@ def test_chat_over_responses_deployment_folds_commentary_and_final_answer_into_o
"message": {
"role": "assistant",
"content": 'Let me answer that.{"city": "Paris", "summary": "Sunny, 21C"}',
"provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY},
},
}
], response.text
@ -843,6 +850,7 @@ def test_chat_over_responses_deployment_folds_reasoning_two_messages_and_functio
"reasoning_content": "Checking the forecast.",
"reasoning_items": [_REASONING_ITEM],
"tool_calls": _TWO_CALLS_CHAT_SHAPE[:1],
"provider_specific_fields": {"openai_responses_replay": ANY},
},
}
], response.text
@ -884,6 +892,7 @@ def test_chat_over_responses_deployment_folds_message_and_two_function_calls_int
"role": "assistant",
"content": "Checking both.",
"tool_calls": _TWO_CALLS_CHAT_SHAPE,
"provider_specific_fields": {"openai_responses_replay": ANY},
},
}
], response.text
@ -1353,7 +1362,11 @@ def test_chat_over_responses_deployment_with_json_schema_returns_first_of_two_fi
{
"finish_reason": "stop",
"index": 0,
"message": {"role": "assistant", "content": '{"city":"Paris"}'},
"message": {
"role": "assistant",
"content": '{"city":"Paris"}',
"provider_specific_fields": {"phase": "final_answer"},
},
}
], response.text
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/responses")]

View file

@ -39,14 +39,6 @@ _CASES: Final[tuple[tuple[str, Mode, JsonValue, JsonValue], ...]] = (
)
_CASE_IDS: Final = tuple(case[0] for case in _CASES)
_CASE_VALUES: Final = tuple(case[1:] for case in _CASES)
_ADAPTER_CASES: Final[tuple[tuple[str, Mode, JsonValue, JsonValue], ...]] = (
("valid-on", "on", pcb.EXPLICIT, pcb.EXPLICIT),
("valid-off", "off", pcb.EXPLICIT, pcb.EXPLICIT),
("malformed-on", "on", "yes", "yes"),
("malformed-off", "off", "yes", "yes"),
)
_ADAPTER_CASE_IDS: Final = tuple(case[0] for case in _ADAPTER_CASES)
_ADAPTER_CASE_VALUES: Final = tuple(case[1:] for case in _ADAPTER_CASES)
@dataclass(frozen=True, slots=True)
@ -331,7 +323,7 @@ def _anthropic_text(message: anthropic.types.Message, marker: str) -> None:
assert content.type == "text" and content.text == rv.answer(marker), message
@pytest.mark.parametrize(("mode", "breakpoint", "expected"), _ADAPTER_CASE_VALUES, ids=_ADAPTER_CASE_IDS)
@pytest.mark.parametrize(("mode", "breakpoint", "expected"), _CASE_VALUES, ids=_CASE_IDS)
def test_anthropic_sdk_marker_on_user_text(
bridge: _Bridge, mode: Mode, breakpoint: JsonValue, expected: JsonValue
) -> None:
@ -540,14 +532,17 @@ def test_anthropic_sdk_cache_control_on_an_assistant_turn_is_dropped(bridge: _Br
], "OpenAI accepts a breakpoint on output_text but writes no cache for it (live gpt-5.6, 2026-10-12)"
def test_native_responses_request_never_enters_the_bridge(bridge: _Bridge) -> None:
@pytest.mark.parametrize(
("mode", "expected"), (("on", pcb.EXPLICIT), ("off", _UNKNOWN_KEY)), ids=("normalized-on", "verbatim-off")
)
def test_native_responses_marker_with_an_unknown_key(bridge: _Bridge, mode: Mode, expected: JsonValue) -> None:
marker: Final = uuid.uuid4().hex
block: Final[dict[str, JsonValue]] = {"type": "input_text", "text": pcb.prompt(marker)}
response: Final = bridge.gateway.request(
"POST",
"/v1/responses",
{
"model": bridge.on,
"model": bridge.model(mode),
"input": [{"type": "message", "role": "user", "content": [pcb.marked(block, _UNKNOWN_KEY)]}],
**pcb.NO_CACHE,
},
@ -559,8 +554,31 @@ def test_native_responses_request_never_enters_the_bridge(bridge: _Bridge) -> No
request: Final = pcb.posted(bridge.wire, marker)
_wire_body(request)
on_wire: Final = pcb.single_block(pcb.input_items(request), "user")
assert on_wire == pcb.marked(block, _UNKNOWN_KEY), on_wire
bridge.spend.landed(bridge.on, response.headers["x-litellm-call-id"], marker)
assert on_wire == pcb.marked(block, expected), on_wire
bridge.spend.landed(bridge.model(mode), response.headers["x-litellm-call-id"], marker)
@pytest.mark.parametrize(("mode", "breakpoint", "expected"), _CASE_VALUES, ids=_CASE_IDS)
def test_openai_sdk_native_responses_marker_on_user_text(
bridge: _Bridge, mode: Mode, breakpoint: JsonValue, expected: JsonValue
) -> None:
marker: Final = uuid.uuid4().hex
block: Final[dict[str, JsonValue]] = {"type": "input_text", "text": pcb.prompt(marker)}
with openai.OpenAI(api_key=bridge.gateway.key, base_url=_v1(bridge.gateway), max_retries=0) as client:
raw: Final = client.responses.with_raw_response.create(
model=bridge.model(mode),
input=[{"type": "message", "role": "user", "content": [pcb.marked(block, breakpoint)]}],
extra_body=dict(pcb.NO_CACHE),
)
response: Final = raw.parse()
assert pcb.answers(response.id, marker), response
assert response.output_text == rv.answer(marker), response
request: Final = pcb.posted(bridge.wire, marker)
_wire_body(request)
on_wire: Final = pcb.single_block(pcb.input_items(request), "user")
assert on_wire["type"] == "input_text" and on_wire["text"] == pcb.prompt(marker), on_wire
pcb.assert_marker(on_wire, expected)
bridge.spend.landed(bridge.model(mode), raw.headers["x-litellm-call-id"], marker)
@pytest.mark.parametrize("breakpoint", _MALFORMED_VALUES, ids=_MALFORMED_IDS)

View file

@ -0,0 +1,204 @@
import json
import uuid
from collections.abc import Mapping
from typing import Final
import httpx
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
BEDROCK_GPT: Final = "us.openai.gpt-5.6-sol"
MANTLE_MODEL: Final = "bedrock_mantle/openai.gpt-5.6-sol"
TOKEN: Final = "synthetic-bedrock-bearer"
JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
TOOL_SPECS: Final = TypeAdapter(tuple[dict[str, dict[str, JsonValue]], ...])
def function_tool(name: str, description: str = "") -> dict[str, JsonValue]:
return {
"type": "function",
"name": name,
"description": description or f"run {name}",
"parameters": {"type": "object", "properties": {"arg": {"type": "string"}}},
}
def namespace_tool(name: str, *members: dict[str, JsonValue]) -> dict[str, JsonValue]:
return {"type": "namespace", "name": name, "description": "", "tools": list(members)}
def tool_list_item(*tools: dict[str, JsonValue]) -> dict[str, JsonValue]:
return {"type": "additional_tools", "role": "developer", "tools": list(tools)}
def user_turn(text: str) -> dict[str, JsonValue]:
return {"type": "message", "role": "user", "content": [{"type": "input_text", "text": text}]}
def assistant_turn(text: str) -> dict[str, JsonValue]:
return {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": text}]}
def converse_tool_specs(request: Request) -> tuple[dict[str, JsonValue], ...]:
tool_config: Final = JSON_OBJECT.validate_json(request.body).get("toolConfig")
assert isinstance(tool_config, dict), request.body
return tuple(entry["toolSpec"] for entry in TOOL_SPECS.validate_python(tool_config["tools"]))
def bedrock_converse_peer(request: Request) -> Reply:
names: Final = tuple(str(spec["name"]) for spec in converse_tool_specs(request))
repeated: Final = tuple(name for index, name in enumerate(names) if name in names[:index])
if repeated:
return Reply(
status=400,
body=json.dumps(
{
"message": f"The tool {repeated[0]} is already defined at toolConfig.tools.{names.index(repeated[0])}."
}
).encode(),
)
return Reply(
body=json.dumps(
{
"output": {"message": {"role": "assistant", "content": [{"text": "converse wire control"}]}},
"stopReason": "end_turn",
"usage": {"inputTokens": 30, "outputTokens": 3, "totalTokens": 33},
"metrics": {"latencyMs": 10},
}
).encode()
)
def send_to_bedrock_converse(gateway: Gateway, history: list[JsonValue]) -> tuple[httpx.Response, tuple[Request, ...]]:
with wire_server(bedrock_converse_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"bedrock/converse/{BEDROCK_GPT}",
api_key=TOKEN,
aws_region_name="us-east-1",
aws_bedrock_runtime_endpoint=wire.url,
)
response: Final = gateway.request(
"POST", "/v1/responses", {"model": model, "input": history, "store": False, "cache": {"no-cache": True}}
)
return response, wire.drain()
def forwarded_specs_by_name(forwarded: tuple[Request, ...]) -> Mapping[str, dict[str, JsonValue]]:
assert len(forwarded) == 1, forwarded
return {str(spec["name"]): spec for spec in converse_tool_specs(forwarded[0])}
def test_codex_tool_list_repeated_on_every_turn_reaches_bedrock_converse_once(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
tools: Final = tool_list_item(
namespace_tool("collaboration", function_tool("followup_task")), function_tool("exec_command")
)
history: Final[list[JsonValue]] = [
tools,
user_turn(f"first {marker}"),
assistant_turn("ok"),
tools,
user_turn(f"second {marker}"),
assistant_turn("ok"),
tools,
user_turn(f"third {marker}"),
]
response, forwarded = send_to_bedrock_converse(gateway, history)
assert response.status_code == 200, response.text
assert response.json()["output"][0]["content"][0]["text"] == "converse wire control", response.text
names: Final = [str(spec["name"]) for spec in converse_tool_specs(forwarded[0])]
assert sorted(names) == ["collaboration__followup_task", "exec_command"], names
def test_newest_copy_of_a_repeated_codex_tool_is_what_bedrock_converse_receives(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
history: Final[list[JsonValue]] = [
tool_list_item(function_tool("exec_command", "old description")),
user_turn(f"first {marker}"),
assistant_turn("ok"),
tool_list_item(function_tool("exec_command", "new description")),
user_turn(f"second {marker}"),
]
response, forwarded = send_to_bedrock_converse(gateway, history)
assert response.status_code == 200, response.text
specs: Final = forwarded_specs_by_name(forwarded)
assert list(specs) == ["exec_command"], specs
assert specs["exec_command"]["description"] == "new description", specs
def test_distinct_codex_tools_spread_across_turns_all_reach_bedrock_converse(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
history: Final[list[JsonValue]] = [
tool_list_item(function_tool("exec_command"), namespace_tool("mcp__docs__", function_tool("search"))),
user_turn(f"first {marker}"),
assistant_turn("ok"),
tool_list_item(function_tool("apply_patch"), namespace_tool("mcp__docs__", function_tool("fetch"))),
user_turn(f"second {marker}"),
]
response, forwarded = send_to_bedrock_converse(gateway, history)
assert response.status_code == 200, response.text
assert sorted(forwarded_specs_by_name(forwarded)) == [
"apply_patch",
"exec_command",
"mcp__docs____fetch",
"mcp__docs____search",
]
def test_codex_tool_list_repeated_on_every_turn_reaches_bedrock_mantle_once(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
shell: Final = function_tool("shell")
collaboration: Final = namespace_tool("collaboration", function_tool("followup_task"))
def mantle_peer(request: Request) -> Reply:
return Reply(
body=json.dumps(
{
"id": "resp_synthetic_mantle",
"object": "response",
"created_at": 1789788253,
"status": "completed",
"model": "openai.gpt-5.6-sol",
"output": [
{
"type": "message",
"id": "msg_synthetic_mantle",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "mantle wire control", "annotations": []}],
}
],
"usage": {"input_tokens": 21, "output_tokens": 4, "total_tokens": 25},
}
).encode()
)
with wire_server(mantle_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=MANTLE_MODEL, api_key=TOKEN, api_base=wire.url, aws_region_name="us-east-2")
response: Final = gateway.request(
"POST",
"/v1/responses",
{
"model": model,
"input": [
tool_list_item(shell, collaboration),
user_turn(f"first {marker}"),
assistant_turn("ok"),
tool_list_item(shell, collaboration),
user_turn(f"second {marker}"),
],
"store": False,
},
)
forwarded: Final = wire.drain()
assert response.status_code == 200, response.text
assert len(forwarded) == 1, forwarded
assert JSON_OBJECT.validate_json(forwarded[0].body)["tools"] == [shell, collaboration], forwarded[0].body

View file

@ -2,17 +2,21 @@ from __future__ import annotations
import json
import os
import threading
import uuid
from collections.abc import Callable
from types import MappingProxyType
from typing import Final, Literal
from urllib.parse import parse_qsl, urlsplit
import httpx
import openai
import pytest
from integration._support.client import Gateway, Scenario, eventually, object_value, string_value
from integration._support.responses_vendor import same_response
from integration._support.wire import Reply, Request, Wire, wire_server
from integration._support.worker_readiness import wait_for_worker_readiness
from openai.types.responses import Response, ResponseStreamEvent
from openai.types.responses import Response, ResponseCreatedEvent, ResponseStreamEvent
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
_MODEL: Final = "gpt-5"
@ -87,11 +91,14 @@ def _stream(identity: str) -> tuple[bytes, ...]:
) + (b"data: [DONE]\n\n",)
def _deployment(scenario: Scenario, wire: Wire, api_key: str = _API_KEY) -> str:
def _deployment(
scenario: Scenario, wire: Wire, api_key: str = _API_KEY, provider: str = "openai", **parameters: JsonValue
) -> str:
model: Final = scenario.model(
model=f"openai/{_MODEL}",
model=f"{provider}/{_MODEL}",
api_key=api_key,
api_base=f"{wire.url}/v1",
**parameters,
)
_wait_for_workers(scenario.gateway, model, 1)
return model
@ -297,9 +304,6 @@ def test_streamed_response_ids_are_stable_and_retrievable_on_each_alias(
@pytest.mark.parametrize("prefix", _ALIASES)
def test_retrieve_query_reaches_the_provider(gateway: Gateway, prefix: str) -> None:
pytest.skip(
"BUG: GET /v1/responses/{id} drops include and starting_after; streamed GET returns 500 for provider SSE"
)
normal_raw_id: Final = f"resp_query_normal_{uuid.uuid4().hex}"
stream_raw_id: Final = f"resp_query_stream_{uuid.uuid4().hex}"
normal_include: Final = ["message.output_text.logprobs", "reasoning.encrypted_content"]
@ -413,6 +417,241 @@ def test_retrieve_query_reaches_the_provider(gateway: Gateway, prefix: str) -> N
assert actual_queries == expected_queries, requests
def _resumed_delta() -> bytes:
event: Final = {
"type": "response.output_text.delta",
"sequence_number": 4,
"item_id": "msg_resumed",
"output_index": 0,
"content_index": 0,
"delta": "resumed",
"logprobs": [],
}
return f"data: {json.dumps(event)}\n\n".encode()
def _sorted_query(request: Request) -> tuple[tuple[str, str], ...]:
return tuple(sorted(parse_qsl(urlsplit(request.target).query, keep_blank_values=True)))
@pytest.mark.parametrize("prefix", _ALIASES)
def test_resuming_past_the_last_event_ends_the_stream_without_events(gateway: Gateway, prefix: str) -> None:
raw_id: Final = f"resp_resume_end_{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
if request.method == "GET" and request.target == "/v1/models":
return model_discovery_reply(_MODEL)
if request.method == "POST":
return Reply(body=_response(raw_id))
return Reply(content_type="text/event-stream", chunks=())
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
client: Final = _client(gateway, gateway.key, prefix)
created: Final = client.responses.create(model=model, input="resume past the end")
events: Final = tuple(client.responses.retrieve(created.id, stream=True, starting_after=7))
requests: Final = drain_contract_requests(wire)
assert events == ()
assert [(request.method, urlsplit(request.target).path) for request in requests] == [
("POST", "/v1/responses"),
("GET", f"/v1/responses/{raw_id}"),
], requests
assert _sorted_query(requests[1]) == (("starting_after", "7"), ("stream", "true")), requests[1]
@pytest.mark.parametrize("prefix", _ALIASES)
def test_a_client_leaving_a_resumed_stream_releases_the_provider_connection(gateway: Gateway, prefix: str) -> None:
raw_id: Final = f"resp_resume_leave_{uuid.uuid4().hex}"
release: Final = threading.Event()
def respond(request: Request) -> Reply:
if request.method == "GET" and request.target == "/v1/models":
return model_discovery_reply(_MODEL)
if request.method == "POST":
return Reply(body=_response(raw_id))
return Reply(
content_type="text/event-stream",
chunks=(_resumed_delta(), b"data: [DONE]\n\n"),
gate_after_first=release,
heartbeat=b": keepalive\n\n",
pause_between_chunks=0.05,
)
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
client: Final = _client(gateway, gateway.key, prefix)
created: Final = client.responses.create(model=model, input="leave a resumed stream")
stream: Final = client.responses.retrieve(created.id, stream=True, starting_after=3)
try:
first: Final = next(iter(stream))
stream.close()
left_from: Final = wire.disconnected.get(timeout=30)
finally:
release.set()
assert first.type == "response.output_text.delta"
assert urlsplit(left_from).path == f"/v1/responses/{raw_id}"
def _created_then(raw_id: str, read: Reply) -> Callable[[Request], Reply]:
def respond(request: Request) -> Reply:
if request.method == "GET" and request.target == "/v1/models":
return model_discovery_reply(_MODEL)
if request.method == "POST":
return Reply(body=_response(raw_id))
return read
return respond
def _event_types(answer: httpx.Response) -> list[JsonValue]:
return [
_JSON_OBJECT.validate_json(line.removeprefix("data: "))["type"]
for line in answer.text.splitlines()
if line.startswith("data: {")
]
_MALFORMED_RESUMES: Final = MappingProxyType(
{
"stream": {"stream": "maybe"},
"starting_after": {"stream": "true", "starting_after": "-1"},
}
)
@pytest.mark.parametrize("param", tuple(_MALFORMED_RESUMES))
@pytest.mark.parametrize("prefix", _ALIASES)
def test_a_malformed_resume_query_is_refused_without_calling_the_provider(
gateway: Gateway, prefix: str, param: str
) -> None:
raw_id: Final = f"resp_malformed_{uuid.uuid4().hex}"
replay: Final = Reply(content_type="text/event-stream", chunks=_stream(raw_id))
with wire_server(_created_then(raw_id, replay)) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
created: Final = _client(gateway, gateway.key, prefix).responses.create(model=model, input="malformed resume")
refused: Final = gateway.request("GET", f"{prefix}/responses/{created.id}", params=_MALFORMED_RESUMES[param])
requests: Final = drain_contract_requests(wire)
assert refused.status_code == 400, refused.text
assert object_value(_JSON_OBJECT.validate_json(refused.content)["error"])["param"] == param, refused.text
assert [request.method for request in requests] == ["POST"], requests
@pytest.mark.parametrize("prefix", _ALIASES)
def test_a_malformed_resume_cursor_replays_the_whole_stream_when_drop_params_is_on(
gateway: Gateway, prefix: str
) -> None:
raw_id: Final = f"resp_malformed_dropped_{uuid.uuid4().hex}"
replay: Final = Reply(content_type="text/event-stream", chunks=_stream(raw_id))
with wire_server(_created_then(raw_id, replay)) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire, drop_params=True)
created: Final = _client(gateway, gateway.key, prefix).responses.create(model=model, input="dropped cursor")
replayed: Final = gateway.request(
"GET", f"{prefix}/responses/{created.id}", params=_MALFORMED_RESUMES["starting_after"]
)
requests: Final = drain_contract_requests(wire)
assert replayed.status_code == 200, replayed.text
assert _event_types(replayed) == ["response.created", "response.completed"], replayed.text
assert [request.method for request in requests] == ["POST", "GET"], requests
assert _sorted_query(requests[1]) == (("stream", "true"),), requests[1]
@pytest.mark.parametrize("prefix", _ALIASES)
def test_resuming_on_a_provider_that_cannot_replay_is_refused_without_calling_it(gateway: Gateway, prefix: str) -> None:
raw_id: Final = f"resp_no_replay_{uuid.uuid4().hex}"
with wire_server(_created_then(raw_id, Reply(body=_response(raw_id)))) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire, provider="litellm_proxy")
created: Final = _client(gateway, gateway.key, prefix).responses.create(model=model, input="cannot replay")
refused: Final = gateway.request(
"GET", f"{prefix}/responses/{created.id}", params={"stream": "true", "starting_after": "3"}
)
requests: Final = drain_contract_requests(wire)
assert refused.status_code == 400, refused.text
message: Final = string_value(object_value(_JSON_OBJECT.validate_json(refused.content)["error"])["message"])
assert "litellm_proxy does not support streaming a stored response" in message, refused.text
assert [request.method for request in requests] == ["POST"], requests
@pytest.mark.parametrize("prefix", _ALIASES)
def test_resuming_on_a_provider_that_cannot_replay_answers_the_stored_response_when_drop_params_is_on(
gateway: Gateway, prefix: str
) -> None:
raw_id: Final = f"resp_no_replay_dropped_{uuid.uuid4().hex}"
with wire_server(_created_then(raw_id, Reply(body=_response(raw_id)))) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire, provider="litellm_proxy", drop_params=True)
created: Final = _client(gateway, gateway.key, prefix).responses.create(model=model, input="dropped replay")
answered: Final = gateway.request(
"GET", f"{prefix}/responses/{created.id}", params={"stream": "true", "starting_after": "3"}
)
requests: Final = drain_contract_requests(wire)
assert answered.status_code == 200, answered.text
body: Final = _JSON_OBJECT.validate_json(answered.content)
assert (body["object"], body["status"], same_response(string_value(body["id"]), created.id)) == (
"response",
"completed",
True,
), answered.text
assert [request.method for request in requests] == ["POST", "GET"], requests
assert _sorted_query(requests[1]) == (), requests[1]
@pytest.mark.parametrize("prefix", _ALIASES)
def test_a_resumed_stream_waits_out_the_stream_timeout_not_the_request_timeout(gateway: Gateway, prefix: str) -> None:
raw_id: Final = f"resp_slow_replay_{uuid.uuid4().hex}"
slow: Final = Reply(content_type="text/event-stream", chunks=_stream(raw_id), pause_between_chunks=2)
with wire_server(_created_then(raw_id, slow)) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire, timeout=1, stream_timeout=30)
client: Final = _client(gateway, gateway.key, prefix)
created: Final = client.responses.create(model=model, input="slow replay")
events: Final = tuple(client.responses.retrieve(created.id, stream=True, starting_after=3))
assert [event.type for event in events] == ["response.created", "response.completed"], events
@pytest.mark.parametrize("prefix", _ALIASES)
def test_a_plain_retrieve_on_a_deployment_set_to_stream_answers_the_response_object(
gateway: Gateway, prefix: str
) -> None:
raw_id: Final = f"resp_streaming_deployment_{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
if request.method == "GET" and request.target == "/v1/models":
return model_discovery_reply(_MODEL)
if request.method == "POST":
return Reply(content_type="text/event-stream", chunks=_stream(raw_id))
return Reply(body=_response(raw_id))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire, stream=True)
events: Final = tuple(
_client(gateway, gateway.key, prefix).responses.create(model=model, input="forced stream", stream=True)
)
created_id: Final = next(event.response.id for event in events if isinstance(event, ResponseCreatedEvent))
retrieved: Final = gateway.request("GET", f"{prefix}/responses/{created_id}")
requests: Final = drain_contract_requests(wire)
assert retrieved.status_code == 200, retrieved.text
assert retrieved.headers["content-type"].startswith("application/json"), retrieved.headers
body: Final = _JSON_OBJECT.validate_json(retrieved.content)
assert (body["object"], body["status"], same_response(string_value(body["id"]), created_id)) == (
"response",
"completed",
True,
), retrieved.text
assert [request.method for request in requests] == ["POST", "GET"], requests
assert _sorted_query(requests[1]) == (), requests[1]
@pytest.mark.parametrize("prefix", _ALIASES)
def test_delete_forwards_the_provider_id_once_and_reports_deletion(
gateway: Gateway,

View file

@ -0,0 +1,37 @@
import uuid
from typing import Final, cast
from integration._support.responses_replay import (
ANSWER,
MODEL,
WEATHER_REPLAY,
WEATHER_TOOL,
WEATHER_TURN,
posted_inputs,
responses_upstream,
)
from integration._support.wire import wire_server
import litellm
from litellm.types.llms.openai import ChatCompletionToolMessage, ChatCompletionToolParam
from litellm.types.utils import ChatCompletionMessageToolCall
def _rain(call: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage:
return {"role": "tool", "tool_call_id": call.id, "content": "Rain, 18C"}
def test_run_tool_loop_replays_responses_items_in_the_order_openai_produced() -> None:
prompt: Final = f"Pick a city and check whether it is raining there {uuid.uuid4().hex}"
with wire_server(responses_upstream(WEATHER_TURN)) as wire:
content: Final = litellm.run_tool_loop(
model=f"openai/responses/{MODEL}",
messages=[{"role": "user", "content": prompt}],
tools=[cast(ChatCompletionToolParam, WEATHER_TOOL)],
execute_tool=_rain,
api_base=wire.url,
api_key="synthetic-openai-key",
)
inputs: Final = posted_inputs(wire)
assert content == ANSWER
assert inputs[1] == (("message", "user", None, prompt), *WEATHER_REPLAY), inputs

View file

@ -7,6 +7,7 @@ from collections.abc import Callable, Mapping
from concurrent.futures import ThreadPoolExecutor
from itertools import chain
from typing import Final
from urllib.parse import parse_qsl, urlsplit
import httpx
import openai
@ -24,7 +25,7 @@ from integration._support.client import (
from integration._support.database import read_rows, write_rows
from integration._support.otlp_sink import Span, decode_spans
from integration._support.upstream import delete_scenario, register_scenario
from integration._support.wire import Request, wire_server
from integration._support.wire import Reply, Request, wire_server
from integration.caching.response_cache_case import entry_key, redis_store
from integration.security._callback_traffic import langfuse_sink
from integration.security._sinks import Recorder
@ -67,6 +68,7 @@ from integration.spend._background_responses import (
scripted_routes,
scripted_upstream,
settling_reads,
sse_frame,
streamed_reply,
submissions,
tag_spend,
@ -350,6 +352,45 @@ def test_a_streamed_background_response_read_later_bills_its_stream_alone(gatewa
assert_charged_exactly(lambda: key_spend(key), PRICE)
def _finishes_only_when_resumed(raw_id: str) -> Callable[[Request], Reply]:
finished: Final = (sse_frame("response.completed", 9, response_body("completed", id=raw_id)) + "\n\n").encode()
queued: Final = json.dumps(response_body("queued", id=raw_id)).encode()
def respond(request: Request) -> Reply:
if urlsplit(request.target).path == "/v1/models":
return Reply(body=b'{"object": "list", "data": []}')
if request.method == "GET" and ("stream", "true") in parse_qsl(urlsplit(request.target).query):
return Reply(content_type="text/event-stream", chunks=(finished, b"data: [DONE]\n\n"))
return Reply(body=queued)
return respond
@pytest.mark.timeout(240)
def test_a_background_response_first_seen_finished_on_a_resumed_stream_bills_once(gateway: Gateway) -> None:
raw_id: Final = f"resp_resumed_{uuid.uuid4().hex}"
with wire_server(_finishes_only_when_resumed(raw_id)) as wire, gateway.scenario() as scenario:
key: Final = scenario.key()
model: Final = scenario.model(
model="openai/gpt-4o-mini",
api_key="sk-scripted-provider",
api_base=f"{wire.url}/v1",
input_cost_per_token=INPUT_COST_PER_TOKEN,
output_cost_per_token=OUTPUT_COST_PER_TOKEN,
)
response_id: Final = create_background_response(scenario, key, model)
resumed: Final = tuple(
gateway.request(
"GET", f"/v1/responses/{response_id}", key=key, params={"stream": "true", "starting_after": "0"}
)
for _ in range(READS)
)
assert [(answer.status_code, "response.completed" in answer.text) for answer in resumed] == [
(200, True)
] * READS, [answer.text for answer in resumed]
assert_charged_exactly(lambda: key_spend(key), PRICE)
def _finished(scenario: Scenario, key: str) -> tuple[Upstream, str]:
upstream: Final = scripted_upstream(scenario, background_routes("completed"))
return upstream, create_background_response(scenario, key, upstream.model)

View file

@ -97,7 +97,17 @@ GPT_5_6_SOL_TEST_CASE: Final = TranslationTestCase(
"created": ANY,
"model": "openai/responses/gpt-5.6-sol",
"object": "chat.completion",
"choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Hello!", "role": "assistant"}}],
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"content": "Hello!",
"role": "assistant",
"provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY},
},
}
],
"usage": {
"completion_tokens": 6,
"prompt_tokens": 19,
@ -252,6 +262,7 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase(
"summary": [],
}
],
"provider_specific_fields": {"phase": "commentary", "openai_responses_replay": ANY},
},
}
],

View file

@ -57,7 +57,10 @@ def two_messages_before_tools(case: TranslationTestCase) -> TranslationTestCase:
def _messages_then_tools_response(case: TranslationTestCase) -> Mapping[str, JsonValue]:
expected: Final = case.expected_litellm_response
if case.litellm_endpoint == "/v1/chat/completions":
choice: Final = items(expected["choices"])[0]
choices: Final = expected["choices"]
assert isinstance(choices, list) and isinstance(choices[0], dict), choices
message: Final = choices[0]["message"]
assert isinstance(message, dict), message
return {
**expected,
"choices": [
@ -67,7 +70,12 @@ def _messages_then_tools_response(case: TranslationTestCase) -> Mapping[str, Jso
"message": {
"role": "assistant",
"content": "".join(COMMENTARY),
"tool_calls": object_value(choice["message"])["tool_calls"],
"tool_calls": message["tool_calls"],
**(
{"provider_specific_fields": message["provider_specific_fields"]}
if "provider_specific_fields" in message
else {}
),
},
}
],

View file

@ -473,6 +473,12 @@ ADVISOR_TOOL: Final = {"type": ANTHROPIC_ADVISOR_TOOL_TYPE, "name": "advisor", "
{"messages": [{"role": "assistant", "content": [{"type": "redacted_thinking", "data": "x"}]}]},
{},
),
(
Feature.TOOL_CUSTOM_FIELD,
"anthropic/claude-sonnet-4-5",
{"tools": [{"name": "a", "input_schema": {"type": "object"}, "custom": {"eager_input_streaming": True}}]},
{},
),
(
Feature.STREAM_COST_INJECTION,
"anthropic/claude-sonnet-4-5",

View file

@ -0,0 +1,627 @@
import json
from collections.abc import Callable, Iterator, Mapping, Sequence
from typing import Final
import httpx
import pytest
import respx
import litellm
from litellm.completion_extras.litellm_responses_transformation import history
from litellm.exceptions import MidStreamFallbackError
OPENAI_RESPONSES_URL: Final = "https://api.openai.com/v1/responses"
AZURE_API_BASE: Final = "https://litellm-history-test.openai.azure.com"
MODEL: Final = "gpt-5.6"
TOOLS: Final = (
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
},
},
)
USER: Final = {"role": "user", "content": "Weather in Paris? Narrate as you go."}
COMMENTARY: Final = {
"type": "message",
"id": "msg_commentary",
"status": "completed",
"role": "assistant",
"phase": "commentary",
"content": [{"type": "output_text", "text": "Checking the weather.", "annotations": []}],
}
FIRST_REASONING: Final = {
"type": "reasoning",
"id": "rs_first",
"summary": [{"type": "summary_text", "text": "Paris is likely wet."}],
"encrypted_content": "enc_first",
}
SECOND_REASONING: Final = {"type": "reasoning", "id": "rs_second", "summary": [], "encrypted_content": "enc_second"}
WEATHER_CALL: Final = {
"type": "function_call",
"id": "fc_weather",
"call_id": "call_weather_paris",
"name": "get_weather",
"arguments": '{"city":"Paris"}',
"status": "completed",
}
FINAL_ANSWER: Final = {
"type": "message",
"id": "msg_final",
"status": "completed",
"role": "assistant",
"phase": "final_answer",
"content": [
{"type": "output_text", "text": "Paris: ", "annotations": []},
{"type": "output_text", "text": "light rain.", "annotations": []},
],
}
OUTPUT: Final = (COMMENTARY, FIRST_REASONING, SECOND_REASONING, WEATHER_CALL, FINAL_ANSWER)
REPLAYED: Final = (
{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "Checking the weather."}],
"phase": "commentary",
},
{
"type": "reasoning",
"id": "rs_first",
"summary": [{"type": "summary_text", "text": "Paris is likely wet."}],
"encrypted_content": "enc_first",
},
{"type": "reasoning", "id": "rs_second", "summary": [], "encrypted_content": "enc_second"},
{"type": "function_call", "call_id": "call_weather_paris", "name": "get_weather", "arguments": '{"city":"Paris"}'},
{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "Paris: "}, {"type": "output_text", "text": "light rain."}],
"phase": "final_answer",
},
)
TOOL_RESULT: Final = {"role": "tool", "tool_call_id": "call_weather_paris", "content": "light rain"}
FOLLOW_UP: Final = {
"type": "message",
"id": "msg_follow_up",
"status": "completed",
"role": "assistant",
"phase": "final_answer",
"content": [{"type": "output_text", "text": "Done.", "annotations": []}],
}
def _response(output: Sequence[Mapping[str, object]], status: str = "completed") -> dict[str, object]:
return {
"id": "resp_history",
"object": "response",
"created_at": 1789788253,
"status": status,
"model": MODEL,
"output": list(output),
"incomplete_details": {"reason": "max_output_tokens"} if status == "incomplete" else None,
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
}
def _added_item(item: Mapping[str, object]) -> Mapping[str, object]:
match item.get("type"):
case "message":
return {**item, "status": "in_progress", "content": []}
case "function_call":
return {**item, "status": "in_progress", "arguments": ""}
case "custom_tool_call":
return {**item, "status": "in_progress", "input": ""}
case _:
return item
def _delta_events(index: int, item: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
match item.get("type"):
case "message":
parts: Final = item["content"]
assert isinstance(parts, list)
return tuple(
{
"type": "response.output_text.delta",
"item_id": item["id"],
"output_index": index,
"content_index": part_index,
"delta": part["text"],
}
for part_index, part in enumerate(parts)
)
case "function_call":
return (
{
"type": "response.function_call_arguments.delta",
"item_id": item["id"],
"output_index": index,
"delta": item["arguments"],
},
)
case "custom_tool_call":
return (
{
"type": "response.custom_tool_call_input.delta",
"item_id": item["id"],
"output_index": index,
"delta": item["input"],
},
)
case _:
return ()
def _item_events(index: int, item: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
return (
{"type": "response.output_item.added", "output_index": index, "item": _added_item(item)},
*_delta_events(index, item),
{"type": "response.output_item.done", "output_index": index, "item": item},
)
def _sse(output: Sequence[Mapping[str, object]], terminal_event: str | None = "response.completed") -> httpx.Response:
item_events: Final = tuple(event for index, item in enumerate(output) for event in _item_events(index, item))
terminal: Final = (
()
if terminal_event is None
else (
{
"type": terminal_event,
"response": _response(output, "completed" if terminal_event == "response.completed" else "incomplete"),
},
)
)
events: Final = (
{"type": "response.created", "response": {**_response((), "in_progress"), "usage": None}},
*item_events,
*terminal,
)
body: Final = "".join(f"data: {json.dumps(event)}\n\n" for event in events)
return httpx.Response(200, content=body, headers={"content-type": "text/event-stream"})
def _assistant_turn(
output: Sequence[Mapping[str, object]],
*,
stream: bool,
model: str = f"openai/responses/{MODEL}",
terminal_event: str | None = "response.completed",
status: str = "completed",
response_format: Mapping[str, object] | None = None,
) -> dict[str, object]:
reply: Final = httpx.Response(
200, content=json.dumps(_response(output, status)), headers={"content-type": "application/json"}
)
with respx.mock(assert_all_called=True) as router:
router.post(url__startswith=OPENAI_RESPONSES_URL if model.startswith("openai/") else AZURE_API_BASE).mock(
return_value=_sse(output, terminal_event) if stream else reply
)
response: Final = litellm.completion(
model=model,
messages=[USER],
tools=list(TOOLS),
api_key="sk-test",
api_base=None if model.startswith("openai/") else AZURE_API_BASE,
api_version=None if model.startswith("openai/") else "2025-04-01-preview",
stream=stream,
response_format=response_format,
)
message: Final = (
(litellm.stream_chunk_builder(list(response), messages=[USER]) if stream else response).choices[0].message
)
return json.loads(json.dumps(message.model_dump()))
def _next_turn_input(
assistant: Mapping[str, object],
model: str = f"openai/responses/{MODEL}",
tools: Sequence[Mapping[str, object]] = TOOLS,
drop_params: bool = False,
) -> list[Mapping[str, object]]:
with respx.mock(assert_all_called=True) as router:
upstream: Final = router.post(
url__startswith=OPENAI_RESPONSES_URL if model.startswith("openai/") else AZURE_API_BASE
).mock(return_value=httpx.Response(200, json=_response((FOLLOW_UP,))))
litellm.completion(
model=model,
messages=[USER, dict(assistant), TOOL_RESULT],
tools=list(tools),
api_key="sk-test",
api_base=None if model.startswith("openai/") else AZURE_API_BASE,
api_version=None if model.startswith("openai/") else "2025-04-01-preview",
drop_params=drop_params,
)
return json.loads(upstream.calls[0].request.content)["input"]
def _replayed_fields(message: Mapping[str, object]) -> Mapping[str, object]:
fields: Final = message.get("provider_specific_fields")
return fields if isinstance(fields, Mapping) else {}
def _types(items: Sequence[Mapping[str, object]]) -> list[object]:
return [item.get("type") for item in items]
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
def test_next_turn_replays_every_item_phase_and_boundary_once_in_order(stream: bool) -> None:
assistant: Final = _assistant_turn(OUTPUT, stream=stream)
sent: Final = _next_turn_input(assistant)
assert assistant["content"] == "Checking the weather.Paris: light rain."
assert _replayed_fields(assistant)["phase"] == "final_answer"
assert sent[1:-1] == list(REPLAYED)
assert sent[-1]["type"] == "function_call_output"
assert sent[-1]["call_id"] == "call_weather_paris"
def _edit_content(message: Mapping[str, object]) -> Mapping[str, object]:
return {**message, "content": "Checking the weather. Paris: heavy rain."}
def _edit_phase(message: Mapping[str, object]) -> Mapping[str, object]:
return {**message, "provider_specific_fields": {**_replayed_fields(message), "phase": "commentary"}}
def _edit_arguments(message: Mapping[str, object]) -> Mapping[str, object]:
tool_calls: Final = message["tool_calls"]
assert isinstance(tool_calls, list)
edited_call: Final = {**tool_calls[0], "function": {**tool_calls[0]["function"], "arguments": '{"city":"Lyon"}'}}
return {**message, "tool_calls": [edited_call]}
@pytest.mark.parametrize(
("edit", "expected_text", "expected_phase", "expected_arguments"),
[
(_edit_content, "Checking the weather. Paris: heavy rain.", "final_answer", '{"city":"Paris"}'),
(_edit_phase, "Checking the weather.Paris: light rain.", "commentary", '{"city":"Paris"}'),
(_edit_arguments, "Checking the weather.Paris: light rain.", "final_answer", '{"city":"Lyon"}'),
],
ids=["content", "phase", "arguments"],
)
def test_an_edited_chat_field_wins_over_the_recorded_items(
edit: Callable[[Mapping[str, object]], Mapping[str, object]],
expected_text: str,
expected_phase: str,
expected_arguments: str,
) -> None:
sent: Final = _next_turn_input(edit(_assistant_turn(OUTPUT, stream=False)))
assert _types(sent[1:-1]) == ["reasoning", "reasoning", "message", "function_call"]
assert sent[3]["content"] == [{"type": "output_text", "text": expected_text}]
assert sent[3]["phase"] == expected_phase
assert sent[4]["arguments"] == expected_arguments
def _without_replay(fields: Mapping[str, object]) -> Mapping[str, object]:
return {key: value for key, value in fields.items() if key != "openai_responses_replay"}
def _with_replay(fields: Mapping[str, object], replay: object) -> Mapping[str, object]:
return {**fields, "openai_responses_replay": replay}
def _replay_of(fields: Mapping[str, object]) -> Mapping[str, object]:
replay: Final = fields["openai_responses_replay"]
assert isinstance(replay, Mapping)
return replay
@pytest.mark.parametrize(
"corrupt",
[
_without_replay,
lambda fields: _with_replay(fields, {**_replay_of(fields), "checksum": "0" * 64}),
lambda fields: _with_replay(fields, {**_replay_of(fields), "version": 2}),
lambda fields: _with_replay(fields, {**_replay_of(fields), "items": [{"type": "message", "parts": [-1]}]}),
lambda fields: _with_replay(fields, "not a replay"),
],
ids=["missing", "checksum", "version", "items", "not-a-mapping"],
)
def test_missing_or_corrupt_replay_metadata_falls_back_to_the_chat_fields(
corrupt: Callable[[Mapping[str, object]], Mapping[str, object]],
) -> None:
assistant: Final = _assistant_turn(OUTPUT, stream=False)
sent: Final = _next_turn_input({**assistant, "provider_specific_fields": corrupt(_replayed_fields(assistant))})
assert _types(sent[1:-1]) == ["reasoning", "reasoning", "message", "function_call"]
assert sent[3]["content"] == [{"type": "output_text", "text": "Checking the weather.Paris: light rain."}]
assert sent[3]["phase"] == "final_answer"
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
def test_an_unphased_answer_after_commentary_is_never_labeled_commentary(stream: bool) -> None:
unphased_answer: Final = {key: value for key, value in FINAL_ANSWER.items() if key != "phase"}
assistant: Final = _assistant_turn((COMMENTARY, WEATHER_CALL, unphased_answer), stream=stream)
sent: Final = _next_turn_input(_edit_content(assistant))
assert "phase" not in _replayed_fields(assistant)
assert _types(sent[1:-1]) == ["message", "function_call"]
assert "phase" not in sent[1]
def _with_text(item: Mapping[str, object], text: str) -> Mapping[str, object]:
return {**item, "content": [{"type": "output_text", "text": text, "annotations": []}]}
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
def test_a_structured_output_turn_carries_the_phase_of_the_message_it_returns(stream: bool) -> None:
output: Final = (_with_text(COMMENTARY, '{"city":"Paris"}'), _with_text(FINAL_ANSWER, '{"city":"Lyon"}'))
assistant: Final = _assistant_turn(output, stream=stream, response_format={"type": "json_object"})
assert assistant["content"] == '{"city":"Paris"}'
assert _replayed_fields(assistant)["phase"] == "commentary"
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
def test_a_lone_surrogate_in_the_reply_still_gets_a_record(stream: bool) -> None:
assistant: Final = _assistant_turn((_with_text(COMMENTARY, "Checking \ud83d"), WEATHER_CALL), stream=stream)
assert "openai_responses_replay" in _replayed_fields(assistant)
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
def test_a_last_message_the_bridge_cannot_read_leaves_the_turn_without_a_phase(stream: bool) -> None:
unreadable: Final = {**_with_text(FINAL_ANSWER, "It is raining in Paris."), "phase": "summary"}
assistant: Final = _assistant_turn((COMMENTARY, WEATHER_CALL, unreadable), stream=stream)
assert "phase" not in _replayed_fields(assistant)
def test_a_phase_openai_does_not_define_is_dropped_under_drop_params() -> None:
assistant: Final = _assistant_turn(OUTPUT, stream=False)
sent: Final = _next_turn_input({**assistant, "provider_specific_fields": {"phase": "analysis"}}, drop_params=True)
assert _types(sent[1:-1]) == ["reasoning", "reasoning", "message", "function_call"]
assert "phase" not in sent[3]
def test_a_phase_openai_does_not_define_is_rejected_without_drop_params() -> None:
assistant: Final = _assistant_turn(OUTPUT, stream=False)
with respx.mock(assert_all_called=False) as router:
upstream: Final = router.post(url__startswith=OPENAI_RESPONSES_URL).mock(
return_value=httpx.Response(200, json=_response((FOLLOW_UP,)))
)
with pytest.raises(litellm.BadRequestError, match="'analysis'"):
litellm.completion(
model=f"openai/responses/{MODEL}",
messages=[USER, {**assistant, "phase": "analysis"}, TOOL_RESULT],
tools=list(TOOLS),
api_key="sk-test",
)
assert not upstream.called
@pytest.mark.parametrize("fields", [None, ["not", "a", "mapping"]], ids=["no-fields", "malformed-fields"])
def test_an_assistant_message_the_app_wrote_replays_without_a_phase(fields: object) -> None:
sent: Final = _next_turn_input(
{
"role": "assistant",
"content": "Checking Paris.",
"tool_calls": [
{
"id": "call_weather_paris",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city": "Paris"}'},
}
],
"provider_specific_fields": fields,
}
)
assert _types(sent[1:-1]) == ["message", "function_call"]
assert "phase" not in sent[1]
SEARCH_CALL: Final = {
"type": "tool_search_call",
"id": "tsc_weather",
"arguments": {"paths": ["get_weather"]},
"status": "completed",
}
SEARCH_OUTPUT: Final = {
"type": "tool_search_output",
"id": "tso_weather",
"status": "completed",
"tools": [{"name": "get_weather"}],
}
@pytest.mark.parametrize(
("tools", "expected"),
[
((*TOOLS, {"type": "tool_search"}), ["tool_search_call", "tool_search_output", *_types(REPLAYED)]),
(TOOLS, ["reasoning", "reasoning", "message", "function_call"]),
],
ids=["searching", "not-searching"],
)
def test_tool_search_items_replay_only_when_the_next_request_searches(
tools: Sequence[Mapping[str, object]], expected: Sequence[str]
) -> None:
assistant: Final = _assistant_turn((SEARCH_CALL, SEARCH_OUTPUT, *OUTPUT), stream=False)
sent: Final = _next_turn_input(assistant, tools=tools)
assert _types(sent[1:-1]) == list(expected)
WEB_SEARCH: Final = {"type": "web_search_call", "id": "ws_1", "status": "completed", "action": {"type": "search"}}
@pytest.mark.parametrize(
("output", "stream", "terminal_event", "status"),
[
(OUTPUT, False, "response.completed", "incomplete"),
(OUTPUT, True, "response.incomplete", "incomplete"),
((WEB_SEARCH, *OUTPUT), False, "response.completed", "completed"),
((WEB_SEARCH, *OUTPUT), True, "response.completed", "completed"),
],
ids=["incomplete-json", "incomplete-stream", "unsupported-item-json", "unsupported-item-stream"],
)
def test_an_incomplete_or_unsupported_output_records_no_replay(
output: Sequence[Mapping[str, object]], stream: bool, terminal_event: str, status: str
) -> None:
assistant: Final = _assistant_turn(output, stream=stream, terminal_event=terminal_event, status=status)
sent: Final = _next_turn_input(assistant)
assert "openai_responses_replay" not in _replayed_fields(assistant)
assert _types(sent[1:-1]) == ["reasoning", "reasoning", "message", "function_call"]
def _chunks_before_error(stream: Iterator[object]) -> Iterator[object]:
try:
yield from stream
except MidStreamFallbackError:
return
def test_an_interrupted_stream_records_no_replay() -> None:
with respx.mock(assert_all_called=True) as router:
router.post(OPENAI_RESPONSES_URL).mock(return_value=_sse(OUTPUT, terminal_event=None))
stream: Final = litellm.completion(
model=f"openai/responses/{MODEL}", messages=[USER], tools=list(TOOLS), api_key="sk-test", stream=True
)
received: Final = tuple(_chunks_before_error(iter(stream)))
assistant: Final = litellm.stream_chunk_builder(list(received), messages=[USER]).choices[0].message.model_dump()
assert assistant["content"] == "Checking the weather.Paris: light rain."
assert _replayed_fields(assistant) == {}
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
def test_a_non_openai_route_records_no_phase_or_replay(stream: bool) -> None:
assistant: Final = _assistant_turn(OUTPUT, stream=stream, model=f"azure/responses/{MODEL}")
assert _replayed_fields(assistant) == {}
def test_a_non_openai_route_ignores_a_recorded_replay() -> None:
sent: Final = _next_turn_input(_assistant_turn(OUTPUT, stream=False), model=f"azure/responses/{MODEL}")
assert _types(sent[1:-1]) == ["reasoning", "reasoning", "message", "function_call"]
assert all("phase" not in item for item in sent)
SECOND_THOUGHT: Final = {**SECOND_REASONING, "summary": [{"type": "summary_text", "text": "Lyon is likely dry."}]}
LYON_CALL: Final = {**WEATHER_CALL, "id": "fc_lyon", "call_id": "call_weather_lyon", "arguments": '{"city":"Lyon"}'}
@pytest.mark.parametrize(
("model", "expected_ids", "expected_text"),
[
(f"openai/responses/{MODEL}", ["rs_first", "rs_second"], "Paris is likely wet. Lyon is likely dry."),
(f"azure/responses/{MODEL}", ["rs_second"], "Lyon is likely dry."),
],
ids=["openai", "azure"],
)
def test_only_the_openai_route_keeps_every_reasoning_item_of_a_tool_call_turn(
model: str, expected_ids: list[str], expected_text: str
) -> None:
assistant: Final = _assistant_turn(
(FIRST_REASONING, WEATHER_CALL, SECOND_THOUGHT, LYON_CALL), stream=False, model=model
)
reasoning_items: Final = assistant["reasoning_items"]
assert isinstance(reasoning_items, list)
assert [item["id"] for item in reasoning_items] == expected_ids
assert assistant["reasoning_content"] == expected_text
@pytest.mark.parametrize(
("model", "expected_content"),
[(f"openai/responses/{MODEL}", None), (f"azure/responses/{MODEL}", "")],
ids=["openai", "azure"],
)
def test_only_the_openai_route_ends_a_streamed_tool_call_turn_with_null_content(
model: str, expected_content: str | None
) -> None:
assistant: Final = _assistant_turn((FIRST_REASONING, WEATHER_CALL), stream=True, model=model)
assert assistant["content"] == expected_content
def _resealed(message: Mapping[str, object], items: Sequence[Mapping[str, object]]) -> Mapping[str, object]:
chat: Final = history._chat_message(message)
assert chat is not None
entries: Final = history._Replay.model_validate({"version": 1, "items": items, "checksum": ""}).items
replay: Final = {"version": 1, "items": list(items), "checksum": history._checksum(chat, entries)}
return {**message, "provider_specific_fields": _with_replay(_replayed_fields(message), replay)}
@pytest.mark.parametrize(
"items",
[
(
{"type": "message", "parts": [21]},
{"type": "function_call", "index": 1},
{"type": "message", "parts": [7, 11]},
),
(
{"type": "message", "parts": [5]},
{"type": "function_call", "index": 0},
{"type": "message", "parts": [7, 11]},
),
],
ids=["tool-call-index", "text-lengths"],
)
def test_a_resealed_record_that_does_not_fit_the_chat_fields_falls_back(items: Sequence[Mapping[str, object]]) -> None:
assistant: Final = _assistant_turn((COMMENTARY, WEATHER_CALL, FINAL_ANSWER), stream=False)
sent: Final = _next_turn_input(_resealed(assistant, items))
assert _types(sent[1:-1]) == ["message", "function_call"]
assert sent[1]["content"] == [{"type": "output_text", "text": "Checking the weather.Paris: light rain."}]
PATCH_CALL: Final = {
"type": "custom_tool_call",
"id": "ctc_patch",
"call_id": "call_patch_readme",
"name": "apply_patch",
"input": "*** Begin Patch\n*** End Patch",
"status": "completed",
}
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
def test_a_custom_tool_call_replays_with_its_custom_output(stream: bool) -> None:
assistant: Final = _assistant_turn((COMMENTARY, PATCH_CALL), stream=stream)
with respx.mock(assert_all_called=True) as router:
upstream: Final = router.post(OPENAI_RESPONSES_URL).mock(
return_value=httpx.Response(200, json=_response((FOLLOW_UP,)))
)
litellm.completion(
model=f"openai/responses/{MODEL}",
messages=[USER, assistant, {"role": "tool", "tool_call_id": "call_patch_readme", "content": "applied"}],
tools=[{"type": "custom", "custom": {"name": "apply_patch"}}],
api_key="sk-test",
)
sent: Final = json.loads(upstream.calls[0].request.content)["input"]
assert sent[1:] == [
{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "Checking the weather."}],
"phase": "commentary",
},
{
"type": "custom_tool_call",
"call_id": "call_patch_readme",
"name": "apply_patch",
"input": PATCH_CALL["input"],
},
{"type": "custom_tool_call_output", "call_id": "call_patch_readme", "output": "applied"},
]

View file

@ -670,6 +670,41 @@ def test_streamed_tool_search_turn_replays_with_its_reasoning_item():
assert request["input"][5]["namespace"] == "get_weather"
@pytest.mark.parametrize(
"answer_item",
[
ResponseFunctionToolCall(
type="function_call", id="fc_1", call_id="call_weather", name="get_weather", arguments="{}"
),
ResponseOutputMessage(
type="message",
id="msg_1",
role="assistant",
status="completed",
content=[ResponseOutputText(type="output_text", text="Rain.", annotations=[])],
),
],
ids=["tool-call", "message"],
)
def test_every_reasoning_item_before_an_answer_reaches_an_openai_chat_message(answer_item: object) -> None:
reasoning: Final = tuple(
ResponseReasoningItem(
type="reasoning",
id=f"rs_{name}",
summary=[Summary(type="summary_text", text=f"{name.title()}.")],
encrypted_content=f"enc_{name}",
)
for name in ("first", "second")
)
message: Final = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices(
[*reasoning, answer_item], replay_openai_history=True
)[0].message
assert [item["id"] for item in message.reasoning_items] == ["rs_first", "rs_second"]
assert message.reasoning_content == "First. Second."
def test_transform_response_with_reasoning_and_output():
"""Test transform_response handles ResponsesAPIResponse with reasoning items and output messages."""
from unittest.mock import Mock
@ -1270,9 +1305,7 @@ def test_transform_request_forwards_defer_loading_only_beside_tool_search(
[{"type": "tool_search", "execution": "client"}, {"type": "tool_search"}],
ids=["client-only", "hosted-and-client"],
)
def test_transform_request_drops_defer_loading_beside_client_tool_search(
deferred_tool, drop_params, tool_search_tool
):
def test_transform_request_drops_defer_loading_beside_client_tool_search(deferred_tool, drop_params, tool_search_tool):
tool_search_tools: Final = (
[tool_search_tool]
if tool_search_tool.get("execution") == "client"
@ -4396,7 +4429,11 @@ def test_tool_search_items_replay_only_to_a_request_that_carries_tool_search(too
"provider_specific_fields": {
"tool_search_items": [
{"type": "tool_search_call", "id": "tsc_1", "arguments": {"paths": ["get_weather"]}},
{"type": "tool_search_output", "id": "tso_1", "tools": [{"type": "function", "name": "get_weather"}]},
{
"type": "tool_search_output",
"id": "tso_1",
"tools": [{"type": "function", "name": "get_weather"}],
},
]
},
},

View file

@ -30,6 +30,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
merge_consecutive_system_messages,
native_redacted_thinking_data,
parse_tool_call_arguments,
prompt_cache_breakpoint_for_wire,
responses_reasoning_items_from_thinking_blocks,
salvage_concatenated_tool_arguments,
schema_grammar_branch_count,
@ -2328,6 +2329,52 @@ def test_get_semantic_cache_prompt_from_messages_matches_get_str_from_messages_w
assert get_semantic_cache_prompt_from_messages(messages) == get_str_from_messages(messages)
_MALFORMED_PROMPT_CACHE_BREAKPOINTS: Final = (
"yes",
1,
["explicit"],
"",
{},
{"mode": "bogus"},
{"mode": "explicit", "ttl": "1h"},
)
_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS: Final = (
"string",
"int",
"list",
"empty-string",
"empty-object",
"bogus-mode",
"bad-ttl",
)
_VALID_PROMPT_CACHE_BREAKPOINTS: Final = ({"mode": "explicit"}, {"mode": "explicit", "ttl": "30m"})
@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep"))
def test_prompt_cache_breakpoint_for_wire_passes_a_missing_marker_through(drop_params: bool) -> None:
assert prompt_cache_breakpoint_for_wire(None, drop_params) is None
@pytest.mark.parametrize("marker", _MALFORMED_PROMPT_CACHE_BREAKPOINTS, ids=_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS)
def test_prompt_cache_breakpoint_for_wire_drops_a_malformed_marker_under_drop_params(marker: object) -> None:
assert prompt_cache_breakpoint_for_wire(marker, True) is None
@pytest.mark.parametrize("marker", _MALFORMED_PROMPT_CACHE_BREAKPOINTS, ids=_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS)
def test_prompt_cache_breakpoint_for_wire_forwards_a_malformed_marker_without_drop_params(marker: object) -> None:
assert prompt_cache_breakpoint_for_wire(marker, False) is marker
@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep"))
@pytest.mark.parametrize("marker", _VALID_PROMPT_CACHE_BREAKPOINTS, ids=("explicit", "explicit-30m"))
def test_prompt_cache_breakpoint_for_wire_keeps_a_valid_marker(marker: dict[str, str], drop_params: bool) -> None:
assert prompt_cache_breakpoint_for_wire(marker, drop_params) == marker
def test_prompt_cache_breakpoint_for_wire_strips_an_unknown_key_under_drop_params() -> None:
assert prompt_cache_breakpoint_for_wire({"mode": "explicit", "note": "kept"}, True) == {"mode": "explicit"}
def test_cache_control_is_replaced_by_an_explicit_prompt_cache_breakpoint_on_the_same_block() -> None:
marked: Final = {"type": "text", "text": "rules", "cache_control": {"type": "ephemeral", "ttl": "1h"}}

View file

@ -9,6 +9,7 @@ from typing import Final, Literal
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import responses
import respx
import litellm
@ -1544,3 +1545,103 @@ async def test_ahealth_check_probes_extraction_models_through_the_extraction_api
sent: Final = json.loads(upstream.calls[0].request.content)
assert sent["state"] == "health check"
assert sent["fields"]["reachable"]["type"] == "text"
VERTEX_AUTHORIZED_USER: Final = json.dumps(
{
"type": "authorized_user",
"client_id": "health-client",
"client_secret": "health-secret",
"refresh_token": "health-refresh",
}
)
CHAT_COMPLETION_BODY: Final = MappingProxyType(
{
"id": "chatcmpl-health",
"object": "chat.completion",
"created": 0,
"model": "health",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
)
GEMINI_GENERATE_CONTENT_BODY: Final = MappingProxyType(
{
"candidates": [{"content": {"role": "model", "parts": [{"text": "ok"}]}, "finishReason": "STOP"}],
"usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2},
}
)
async def test_vertex_wildcard_health_check_only_calls_vertex_hosts(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setenv("OPENAI_API_KEY", "sk-not-vertex")
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
vertex_route: Final = respx_mock.route(host__regex=r".*aiplatform\.googleapis\.com$").respond(
json=dict(GEMINI_GENERATE_CONTENT_BODY)
)
other_hosts: Final = respx_mock.route().respond(json=dict(CHAT_COMPLETION_BODY))
with responses.RequestsMock() as google_token_endpoint:
google_token_endpoint.post(
"https://oauth2.googleapis.com/token",
json={"access_token": "health-access-token", "expires_in": 3600, "token_type": "Bearer"},
)
result: Final = await ahealth_check(
model_params={
"model": "vertex_ai/*",
"vertex_project": "health-project",
"vertex_location": "us-central1",
"vertex_credentials": VERTEX_AUTHORIZED_USER,
},
mode="chat",
)
assert "error" not in result, result
assert vertex_route.called
assert not other_hosts.called
async def test_ai21_wildcard_health_check_calls_ai21(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
ai21_route: Final = respx_mock.route(host="api.ai21.com").respond(json=dict(CHAT_COMPLETION_BODY))
other_hosts: Final = respx_mock.route().respond(json=dict(CHAT_COMPLETION_BODY))
result: Final = await ahealth_check(model_params={"model": "ai21/*", "api_key": "health-ai21-key"}, mode="chat")
assert "error" not in result, result
assert ai21_route.called
assert not other_hosts.called
async def test_wildcard_health_check_with_pinned_provider_calls_a_model_without_a_routing_prefix(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setitem(
litellm.model_cost,
"fake-together-chat",
{"input_cost_per_token": 1e-9, "output_cost_per_token": 1e-9, "mode": "chat", "litellm_provider": "together_ai"},
)
monkeypatch.setitem(litellm.models_by_provider, "together_ai", ("fake-together-chat",))
litellm.in_memory_llm_clients_cache.flush_cache()
together_route: Final = respx_mock.route(host="api.together.ai").respond(json=dict(CHAT_COMPLETION_BODY))
other_hosts: Final = respx_mock.route().respond(json=dict(CHAT_COMPLETION_BODY))
result: Final = await ahealth_check(
model_params={"model": "together_ai/*", "custom_llm_provider": "together_ai", "api_key": "health-together-key"},
mode="chat",
)
assert "error" not in result, result
assert json.loads(together_route.calls.last.request.content)["model"] == "fake-together-chat"
assert not other_hosts.called

View file

@ -181,6 +181,20 @@ class TestNonInferenceCallBilling:
assert is_unbilled_non_inference_call_from_params("aresponses", {}, self.BACKGROUND) is False
assert is_unbilled_non_inference_call_from_params("acompletion", {"metadata": self.MANAGED_READ}, None) is False
def test_a_streamed_read_prices_like_a_plain_read_of_the_same_response(self):
from litellm.types.llms.openai import ResponseCompletedEvent, ResponsesAPIResponse
for stored in (self.BACKGROUND, self.FOREGROUND):
finished = ResponsesAPIResponse(**{**stored, "created_at": 1, "output": []})
completed_event = ResponseCompletedEvent(type="response.completed", response=finished)
for metadata in (None, self.MANAGED_READ, self.STATUS_POLL):
plain_read = is_unbilled_non_inference_call("aget_responses", metadata, finished)
assert is_unbilled_non_inference_call("aget_responses", metadata, completed_event) is plain_read
assert (
is_unbilled_non_inference_call("aget_responses", metadata, completed_event.model_dump())
is plain_read
)
def test_a_tracked_streamed_background_create_is_free_in_every_shape_its_stream_logs(self):
from litellm.types.llms.openai import ResponseCompletedEvent, ResponsesAPIResponse

Some files were not shown because too many files have changed in this diff Show more