fix(anthropic): map /v1/messages fields onto OpenAI Responses equivalents or reject them

The Responses bridge for /v1/messages dropped several Anthropic fields and still answered 200. disable_parallel_tool_use now sets parallel_tool_calls false, speed fast asks for the priority service tier, mcp_servers become Responses mcp tools with their toolset allowlist, web_search keeps its domain filters, user_location, and max_uses, a json_schema output format goes up strict, and a 1h cache_control mark asks OpenAI for its 12h breakpoint TTL

A field with no Responses equivalent (container, allowed_callers, an unknown speed, a toolset denylist, defer_loading, max_uses next to another built-in tool) answers 400 UnsupportedParamsError naming every such field, and drop_params drops them instead

strict_json_schema moves to base_utils so the Codex harness, the chat-completions bridge, and the Responses bridge share one walker, which now leaves property names and enum, const, and examples values alone
This commit is contained in:
mateo-berri 2026-10-29 02:26:27 -07:00
parent ff22f3de06
commit 0ba6bad552
20 changed files with 1131 additions and 172 deletions

View file

@ -25,6 +25,7 @@ from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import PromptManagementClient
from litellm.litellm_core_utils.prompt_templates.common_utils import (
cache_control_as_prompt_cache_breakpoint,
prompt_cache_breakpoint_for,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
@ -747,7 +748,9 @@ class AnthropicCacheControlHook(CustomPromptManagement):
only the last content block can have cache_control.
"""
if openai_dialect:
return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(message)
return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(
message, PromptCacheBreakpoint(mode="explicit")
)
message_content: Final = message.get("content", None)
@ -780,21 +783,24 @@ class AnthropicCacheControlHook(CustomPromptManagement):
},
)
)
if fields.get("cache_control") is None:
message_mark: Final = fields.get("cache_control")
if message_mark is None:
return renamed
return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(renamed)
return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(
renamed, prompt_cache_breakpoint_for(message_mark)
)
@staticmethod
def _insert_prompt_cache_breakpoint_in_message(message: AllMessageValues) -> AllMessageValues:
def _insert_prompt_cache_breakpoint_in_message(
message: AllMessageValues, breakpoint: PromptCacheBreakpoint
) -> AllMessageValues:
if not _may_carry_prompt_cache_breakpoint(message):
return message
message_content: Final = message.get("content", None)
if isinstance(message_content, str):
marked: Final = copy.copy(message)
marked["content"] = [
with_prompt_cache_breakpoint(
ChatCompletionTextObject(type="text", text=message_content), PromptCacheBreakpoint(mode="explicit")
)
with_prompt_cache_breakpoint(ChatCompletionTextObject(type="text", text=message_content), breakpoint)
]
return marked
if isinstance(message_content, list):
@ -807,9 +813,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
None,
)
if target_index is not None:
message_content[target_index] = with_prompt_cache_breakpoint(
message_content[target_index], PromptCacheBreakpoint(mode="explicit")
)
message_content[target_index] = with_prompt_cache_breakpoint(message_content[target_index], breakpoint)
return message
@staticmethod

View file

@ -2073,13 +2073,24 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key
def prompt_cache_breakpoint_for(cache_control: object) -> PromptCacheBreakpoint:
"""OpenAI's breakpoint for an Anthropic ``cache_control`` mark. A 1h mark asks for at least an hour, and 12h is
the only OpenAI TTL that long. A 5m or unset TTL inherits the request's TTL, 30m by default."""
match cache_control:
case {"ttl": "1h"}:
return PromptCacheBreakpoint(mode="explicit", ttl="12h")
case _:
return PromptCacheBreakpoint(mode="explicit")
def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
"""Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block."""
if block.get("cache_control") is None:
cache_control: Final = block.get("cache_control")
if cache_control is None:
return block
unmarked: Final = {key: value for key, value in block.items() if key != "cache_control"}
return cast( # cast-ok: same block shape as the input with the marker key renamed
_MarkedT, with_prompt_cache_breakpoint(unmarked, PromptCacheBreakpoint(mode="explicit"))
_MarkedT, with_prompt_cache_breakpoint(unmarked, prompt_cache_breakpoint_for(cache_control))
)

View file

@ -1,4 +1,3 @@
import copy
import hashlib
import json
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
@ -16,6 +15,7 @@ from litellm.llms.anthropic.messages.utils import (
json_copy_or_original,
prompt_cache_key_from_user_id,
)
from litellm.llms.base_llm.base_utils import as_json_object, strict_json_schema
from litellm.responses.utils import auto_reasoning_summary
# OpenAI has a 64-character limit for function/tool names
@ -885,59 +885,19 @@ class LiteLLMAnthropicMessagesAdapter:
if output_type != "json_schema":
return None
schema = output_format.get("schema")
schema: Final = as_json_object(json_copy_or_original(output_format.get("schema")))
if not schema:
return None
# Deep copy to avoid mutating the original schema
schema = copy.deepcopy(schema)
# OpenAI strict mode requires additionalProperties: false on every object
self._add_additional_properties_false(schema)
# Convert to OpenAI response_format structure
return {
"type": "json_schema",
"json_schema": {
"name": "structured_output",
"schema": schema,
"schema": strict_json_schema(dict(schema)),
"strict": True,
},
}
@staticmethod
def _add_additional_properties_false(schema: dict) -> None:
"""
Recursively ensure object schemas comply with OpenAI strict mode.
OpenAI's strict mode requires:
1. 'additionalProperties': false at every object nesting level
2. All property keys listed in 'required'
"""
if not isinstance(schema, dict):
return
if schema.get("type") == "object" and "properties" in schema:
schema["additionalProperties"] = False
schema["required"] = list(schema["properties"].keys())
for prop in schema["properties"].values():
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(prop)
# Handle array items
if "items" in schema:
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(schema["items"])
# Handle anyOf/oneOf/allOf
for key in ("anyOf", "oneOf", "allOf"):
if key in schema:
for sub_schema in schema[key]:
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(sub_schema)
# Handle $defs / definitions
for key in ("$defs", "definitions"):
if key in schema:
for def_schema in schema[key].values():
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(def_schema)
def _translate_midturn_system_message_to_openai(
self,
message: AnthropicMessagesSystemMessageParam,

View file

@ -713,6 +713,7 @@ def anthropic_messages_handler(
),
top_k=top_k,
top_p=top_p,
container=container,
_is_async=is_async,
api_key=api_key,
api_base=api_base,

View file

@ -21,6 +21,7 @@ from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
from litellm.types.llms.anthropic import (
AllAnthropicMessageValues,
AllAnthropicToolsValues,
AnthropicMcpServerTool,
AnthropicMessagesRequest,
AnthropicOutputConfig,
AnthropicOutputSchema,
@ -137,6 +138,9 @@ def _build_responses_kwargs(
top_k: int | None = None,
top_p: float | None = None,
output_format: AnthropicOutputSchema | None = None,
speed: str | None = None,
mcp_servers: Sequence[AnthropicMcpServerTool] | None = None,
container: Mapping[str, object] | None = None,
extra_kwargs: Mapping[str, object] | None = None,
) -> dict[str, object]:
"""
@ -179,11 +183,19 @@ def _build_responses_kwargs(
if tool_choice:
request_data["tool_choice"] = tool_choice
if tools:
request_data["tools"] = tools
request_data["tools"] = (
([dict(tool) for tool in _rewritten_blocks(tools, block_rewrite)] or tools) if block_rewrite else tools
)
if top_p is not None:
request_data["top_p"] = top_p
if output_format:
request_data["output_format"] = output_format
if speed is not None:
request_data["speed"] = speed
if mcp_servers:
request_data["mcp_servers"] = list(mcp_servers)
if container is not None:
request_data["container"] = dict(container)
anthropic_request: Final = AnthropicMessagesRequest(**request_data)
responses_kwargs: Final = _ADAPTER.translate_request(
@ -240,6 +252,10 @@ def _build_responses_kwargs(
if explicit_prompt_cache_key is not None:
responses_kwargs["prompt_cache_key"] = explicit_prompt_cache_key
explicit_service_tier: Final = forwarded_kwargs.get("service_tier")
if explicit_service_tier is not None:
responses_kwargs["service_tier"] = explicit_service_tier
deployment_include: Final = forwarded_kwargs.get("include")
bridge_include: Final = responses_kwargs.get("include")
if isinstance(deployment_include, list) and isinstance(bridge_include, list):
@ -276,6 +292,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
top_k: int | None = None,
top_p: float | None = None,
output_format: AnthropicOutputSchema | None = None,
speed: str | None = None,
mcp_servers: Sequence[AnthropicMcpServerTool] | None = None,
container: Mapping[str, object] | None = None,
**kwargs: object,
) -> AnthropicMessagesResponse | AsyncIterator[bytes]:
responses_kwargs: Final = _build_responses_kwargs(
@ -295,6 +314,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
top_k=top_k,
top_p=top_p,
output_format=output_format,
speed=speed,
mcp_servers=mcp_servers,
container=container,
extra_kwargs=kwargs,
)
@ -335,6 +357,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
top_k: int | None = None,
top_p: float | None = None,
output_format: AnthropicOutputSchema | None = None,
speed: str | None = None,
mcp_servers: Sequence[AnthropicMcpServerTool] | None = None,
container: Mapping[str, object] | None = None,
_is_async: bool = False,
**kwargs: object,
) -> (
@ -360,6 +385,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
top_k=top_k,
top_p=top_p,
output_format=output_format,
speed=speed,
mcp_servers=mcp_servers,
container=container,
**kwargs,
)
@ -381,6 +409,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
top_k=top_k,
top_p=top_p,
output_format=output_format,
speed=speed,
mcp_servers=mcp_servers,
container=container,
extra_kwargs=kwargs,
)

View file

@ -7,9 +7,13 @@ path used for OpenAI and Azure models.
import json
from collections.abc import Iterable, Mapping, Sequence
from itertools import groupby
from typing import Any, Final, cast
from dataclasses import dataclass
from itertools import chain, groupby
from typing import Any, Final, TypeAlias, cast
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.litellm_core_utils.prompt_templates.common_utils import (
TOOL_RESULT_IMAGE_BOUNDARY,
TOOL_RESULT_IMAGE_PLACEHOLDER,
@ -30,10 +34,15 @@ from litellm.llms.anthropic.messages.utils import (
json_copy_or_original,
prompt_cache_key_from_user_id,
)
from litellm.llms.base_llm.base_utils import as_json_object, strict_json_schema
from litellm.types.llms.anthropic import (
AllAnthropicPassThroughMessageValues,
AllAnthropicToolsValues,
AnthropicFinishReason,
AnthropicMcpServerTool,
AnthropicMcpServerToolConfiguration,
AnthropicMcpToolConfig,
AnthropicMcpToolset,
AnthropicMessagesRequest,
AnthropicMessagesToolChoice,
AnthropicResponseContentBlockRedactedThinking,
@ -53,6 +62,231 @@ from litellm.types.llms.openai import (
REASONING_SUMMARY_PART_SEPARATOR: Final = "\n\n"
RESPONSES_INCLUDE_ENCRYPTED_REASONING: Final = "reasoning.encrypted_content"
_MCP_TOOLSET_TYPE: Final = "mcp_toolset"
_MCP_TOOLSET: Final = TypeAdapter(AnthropicMcpToolset)
_WEB_SEARCH_FILTER_KEYS: Final = ("allowed_domains", "blocked_domains")
_STRUCTURED_OUTPUT_NAME: Final = "structured_output"
_OBJECT_MAP: Final = TypeAdapter(Mapping[str, object])
@dataclass(frozen=True, slots=True)
class _Translation:
fields: Mapping[str, object]
unsupported: tuple[str, ...] = ()
@dataclass(frozen=True, slots=True)
class _ResponsesTools:
tools: tuple[Mapping[str, object], ...] = ()
unsupported: tuple[str, ...] = ()
@dataclass(frozen=True, slots=True)
class _AllTools:
pass
@dataclass(frozen=True, slots=True)
class _OnlyTools:
names: tuple[str, ...]
@dataclass(frozen=True, slots=True)
class _NoEquivalent:
field: str
_McpAccess: TypeAlias = _AllTools | _OnlyTools | _NoEquivalent
def _present_fields(source: Mapping[str, object], keys: Iterable[str]) -> Mapping[str, object]:
return {key: source[key] for key in keys if source.get(key) is not None}
def _is_web_search_tool(tool: Mapping[str, object]) -> bool:
tool_type: Final = tool.get("type", "")
return (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool.get("name") == "web_search"
def _web_search_tool(tool: Mapping[str, object]) -> dict[str, object]: # mutable-ok: Responses tool payload
"""Domain filters only exist on Responses' ``web_search``, so a filtered search moves off ``web_search_preview``."""
filters: Final = _present_fields(tool, _WEB_SEARCH_FILTER_KEYS)
location: Final = _present_fields(tool, ("user_location",))
if filters:
return {"type": "web_search", "filters": filters, **location}
return {"type": "web_search_preview", **location}
def _allowed_callers_unsupported(tool: Mapping[str, object]) -> tuple[str, ...]:
callers: Final = tool.get("allowed_callers")
if callers is None or callers == ["direct"]:
return ()
return (f"allowed_callers on tool {tool.get('name')!r}",)
def _max_tool_calls(
tools: Sequence[Mapping[str, object]], responses_tools: Sequence[Mapping[str, object]]
) -> _Translation:
"""Responses caps every built-in tool call together, so ``max_uses`` maps only when web search is the one."""
capped: Final = tuple(tool for tool in tools if _is_web_search_tool(tool) and tool.get("max_uses") is not None)
if not capped:
return _Translation(fields={})
built_in: Final = tuple(tool for tool in responses_tools if tool.get("type") != "function")
if len(built_in) == 1:
return _Translation(fields={"max_tool_calls": capped[0].get("max_uses")})
return _Translation(
fields={},
unsupported=tuple(f"max_uses on tool {tool.get('name')!r} next to another built-in tool" for tool in capped),
)
def _configuration_access(configuration: AnthropicMcpServerToolConfiguration | None) -> _McpAccess:
if configuration is None:
return _AllTools()
if configuration.get("enabled") is False:
return _OnlyTools(())
allowed_tools: Final = configuration.get("allowed_tools")
return _AllTools() if allowed_tools is None else _OnlyTools(tuple(allowed_tools))
def _toolset_access(toolset: AnthropicMcpToolset) -> _McpAccess:
default_config: Final[AnthropicMcpToolConfig] = toolset.get("default_config", {})
configs: Final[Mapping[str, AnthropicMcpToolConfig]] = toolset.get("configs", {})
if not default_config.get("enabled", True):
return _OnlyTools(tuple(name for name, config in configs.items() if config.get("enabled") is True))
disabled: Final = tuple(name for name, config in configs.items() if config.get("enabled") is False)
if not disabled:
return _AllTools()
return _NoEquivalent(
f"mcp_toolset {toolset['mcp_server_name']!r} disabling {', '.join(disabled)} while its other tools stay enabled"
)
def _toolset_defer_loading_unsupported(toolset: AnthropicMcpToolset) -> tuple[str, ...]:
default_config: Final[AnthropicMcpToolConfig] = toolset.get("default_config", {})
configs: Final[Mapping[str, AnthropicMcpToolConfig]] = toolset.get("configs", {})
deferred: Final = default_config.get("defer_loading") is True or any(
config.get("defer_loading") is True for config in configs.values()
)
return (f"defer_loading on mcp_toolset {toolset['mcp_server_name']!r}",) if deferred else ()
def _mcp_tool(server: AnthropicMcpServerTool) -> Mapping[str, object]:
"""Anthropic's MCP connector calls tools without asking, so the Responses tool must never wait for an approval."""
connection: Final[Mapping[str, object]] = {
"type": "mcp",
"server_label": server.get("name"),
"server_url": server.get("url"),
"require_approval": "never",
}
authorization: Final = server.get("authorization_token")
return connection if authorization is None else {**connection, "authorization": authorization}
def _mcp_server_tools(
server: AnthropicMcpServerTool, access: _McpAccess, unsupported: tuple[str, ...]
) -> _ResponsesTools:
match access:
case _AllTools():
return _ResponsesTools(tools=(_mcp_tool(server),), unsupported=unsupported)
case _OnlyTools(names=()):
return _ResponsesTools(unsupported=unsupported)
case _OnlyTools(names=names):
return _ResponsesTools(
tools=({**_mcp_tool(server), "allowed_tools": list(names)},), unsupported=unsupported
)
case _NoEquivalent(field=field):
return _ResponsesTools(unsupported=(*unsupported, field))
def _mcp_server_translation(server: AnthropicMcpServerTool, tools: Sequence[Mapping[str, object]]) -> _ResponsesTools:
"""One Responses ``mcp`` tool per server, its access read from the server's ``mcp_toolset`` entry in ``tools``,
else from the deprecated ``tool_configuration``."""
name: Final = server.get("name")
toolset: Final = next(
(tool for tool in tools if tool.get("type") == _MCP_TOOLSET_TYPE and tool.get("mcp_server_name") == name),
None,
)
if toolset is None:
return _mcp_server_tools(server, _configuration_access(server.get("tool_configuration")), ())
try:
parsed: Final = _MCP_TOOLSET.validate_python(toolset)
except ValidationError:
return _mcp_server_tools(server, _NoEquivalent(f"the malformed mcp_toolset for {name!r}"), ())
return _mcp_server_tools(server, _toolset_access(parsed), _toolset_defer_loading_unsupported(parsed))
def _parallel_tool_calls(tool_choice: Mapping[str, object] | None) -> _Translation:
if (
tool_choice is None
or tool_choice.get("type") == "none"
or tool_choice.get("disable_parallel_tool_use") is not True
):
return _Translation(fields={})
return _Translation(fields={"parallel_tool_calls": False})
def _service_tier(speed: object) -> _Translation:
match speed:
case None | "standard":
return _Translation(fields={})
case "fast":
return _Translation(fields={"service_tier": "priority"})
case _:
return _Translation(fields={}, unsupported=(f"speed {speed!r}",))
def _container(container: object) -> _Translation:
return _Translation(fields={}, unsupported=() if container is None else ("container",))
def _object_map(value: object) -> Mapping[str, object] | None:
try:
return _OBJECT_MAP.validate_python(value)
except ValidationError:
return None
def _requested_output_format(request: Mapping[str, object]) -> Mapping[str, object] | None:
output_format: Final = _object_map(request.get("output_format"))
if output_format is not None:
return output_format
output_config: Final = _object_map(request.get("output_config"))
return None if output_config is None else _object_map(output_config.get("format"))
def _text_format(output_format: Mapping[str, object] | None) -> _Translation:
"""A json_schema format goes up strict, as Anthropic structured outputs guarantee the schema. Making every
property required keeps strict output valid against the caller's schema, and an explicit ``strict: false`` or a
schema that is not JSON goes up as is."""
if output_format is None or output_format.get("type") != "json_schema":
return _Translation(fields={})
schema: Final = output_format.get("schema")
if not schema:
return _Translation(fields={})
strict_schema: Final = None if output_format.get("strict") is False else as_json_object(schema)
if strict_schema is None:
loose: Final = {"type": "json_schema", "name": _STRUCTURED_OUTPUT_NAME, "schema": schema, "strict": False}
return _Translation(fields={"text": {"format": loose}})
strict: Final = {
"type": "json_schema",
"name": _STRUCTURED_OUTPUT_NAME,
"schema": strict_json_schema(dict(strict_schema)),
"strict": True,
}
return _Translation(fields={"text": {"format": strict}})
def _unsupported_params_error(model: str, unsupported: Sequence[str]) -> litellm.UnsupportedParamsError:
pronoun: Final = "it" if len(unsupported) == 1 else "them"
return litellm.UnsupportedParamsError(
message=(
f"{model} is served through the OpenAI Responses API, which has no equivalent for "
f"{'; '.join(unsupported)}. To drop {pronoun}, set `litellm.drop_params=True`, or `drop_params: true` "
"under `litellm_settings` on the proxy"
),
model=model,
)
class LiteLLMAnthropicToResponsesAPIAdapter:
@ -466,11 +700,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
@staticmethod
def _translate_tool_to_responses_api(tool_dict: Mapping[str, object]) -> dict[str, object]:
tool_type: Final = tool_dict.get("type", "")
if _is_web_search_tool(tool_dict):
return _web_search_tool(tool_dict)
tool_name: Final = tool_dict.get("name", "")
# web_search tool
if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search":
return {"type": "web_search_preview"}
# Responses turns strict mode on when `strict` is omitted, silently rewriting
# `required` to every property. Anthropic tools are non-strict unless asked.
func_tool: Final[dict[str, object]] = {
@ -482,14 +714,34 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
func_tool["description"] = tool_dict["description"]
if "input_schema" in tool_dict:
func_tool["parameters"] = json_copy_or_original(tool_dict["input_schema"])
return func_tool
return with_prompt_cache_breakpoint(func_tool, tool_dict.get("prompt_cache_breakpoint"))
def translate_tools_to_responses_api(
self,
tools: list[AllAnthropicToolsValues],
) -> list[dict[str, object]]:
"""Convert Anthropic tool definitions to Responses API function tools."""
return [self._translate_tool_to_responses_api(tool) for tool in tools]
"""Convert Anthropic tool definitions to Responses API tools. An ``mcp_toolset`` entry configures its MCP
server's tool, so it never becomes a tool of its own."""
return [self._translate_tool_to_responses_api(tool) for tool in tools if tool.get("type") != _MCP_TOOLSET_TYPE]
def _translate_tools(self, anthropic_request: AnthropicMessagesRequest) -> _Translation:
tools: Final = cast(list[AllAnthropicToolsValues], anthropic_request.get("tools") or [])
mcp_servers: Final = tuple(
_mcp_server_translation(server, tools) for server in anthropic_request.get("mcp_servers") or ()
)
responses_tools: Final = (
*self.translate_tools_to_responses_api(tools),
*chain.from_iterable(server.tools for server in mcp_servers),
)
max_tool_calls: Final = _max_tool_calls(tools, responses_tools)
return _Translation(
fields={**({"tools": list(responses_tools)} if responses_tools else {}), **max_tool_calls.fields},
unsupported=(
*chain.from_iterable(_allowed_callers_unsupported(tool) for tool in tools),
*chain.from_iterable(server.unsupported for server in mcp_servers),
*max_tool_calls.unsupported,
),
)
@staticmethod
def translate_tool_choice_to_responses_api(
@ -653,20 +905,25 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if "top_p" in anthropic_request:
responses_kwargs["top_p"] = anthropic_request["top_p"]
# tools
tools: Final = anthropic_request.get("tools")
if tools:
responses_kwargs["tools"] = self.translate_tools_to_responses_api(
cast(list[AllAnthropicToolsValues], tools)
)
# tool_choice
tool_choice: Final = anthropic_request.get("tool_choice")
if tool_choice:
responses_kwargs["tool_choice"] = self.translate_tool_choice_to_responses_api(
cast(AnthropicMessagesToolChoice, tool_choice)
)
request: Final[Mapping[str, object]] = anthropic_request
translations: Final = (
self._translate_tools(anthropic_request),
_parallel_tool_calls(_object_map(tool_choice)),
_service_tier(request.get("speed")),
_container(request.get("container")),
_text_format(_requested_output_format(request)),
)
unsupported: Final = tuple(chain.from_iterable(translation.unsupported for translation in translations))
if unsupported and not drop_params:
raise _unsupported_params_error(model, unsupported)
responses_kwargs.update(chain.from_iterable(translation.fields.items() for translation in translations))
# thinking -> reasoning
thinking: Final = anthropic_request.get("thinking")
if isinstance(thinking, dict):
@ -680,25 +937,6 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if reasoning:
responses_kwargs["reasoning"] = reasoning
# output_format / output_config.format -> text format
# output_format: {"type": "json_schema", "schema": {...}}
# output_config: {"format": {"type": "json_schema", "schema": {...}}}
output_format: object = anthropic_request.get("output_format")
output_config = anthropic_request.get("output_config")
if not isinstance(output_format, dict) and isinstance(output_config, dict):
output_format = output_config.get("format")
if isinstance(output_format, dict) and output_format.get("type") == "json_schema":
schema: Final = output_format.get("schema")
if schema:
responses_kwargs["text"] = {
"format": {
"type": "json_schema",
"name": "structured_output",
"schema": schema,
"strict": output_format.get("strict", False),
}
}
# context_management: Anthropic dict -> OpenAI array
context_management: Final = anthropic_request.get("context_management")
if isinstance(context_management, dict):

View file

@ -16,6 +16,7 @@ from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage
from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk
from litellm.types.utils import (
@ -196,6 +197,48 @@ def as_json_object(value: object) -> Mapping[str, JsonValue] | None:
return None
_SCHEMA_MAP_KEYWORDS: Final = frozenset({"properties", "$defs", "definitions", "patternProperties"})
_SCHEMA_DATA_KEYWORDS: Final = frozenset({"enum", "const", "examples"})
def _strict_keyword_value(keyword: str, value: JsonValue, depth: int) -> JsonValue:
if keyword in _SCHEMA_DATA_KEYWORDS:
return value
if keyword in _SCHEMA_MAP_KEYWORDS and isinstance(value, dict):
return {name: strict_json_schema(subschema, depth) for name, subschema in value.items()}
return strict_json_schema(value, depth)
def strict_json_schema(schema: JsonValue, depth: int = 0) -> JsonValue:
"""The schema as OpenAI strict structured outputs accept it: every object closed with
`additionalProperties: false` and all of its properties required, `default` removed, and only `$ref` kept on a
reference. Output that matches the result also matches the original. Raises ValueError past
DEFAULT_MAX_RECURSE_DEPTH."""
if depth > DEFAULT_MAX_RECURSE_DEPTH:
raise ValueError(f"output schema is nested deeper than {DEFAULT_MAX_RECURSE_DEPTH} levels")
if isinstance(schema, list):
return [strict_json_schema(entry, depth + 1) for entry in schema]
if not isinstance(schema, dict):
return schema
if "$ref" in schema:
return {"$ref": schema["$ref"]}
walked: Final = {
keyword: _strict_keyword_value(keyword, value, depth + 1)
for keyword, value in schema.items()
if keyword != "default"
}
properties: Final = walked.get("properties")
if walked.get("type") != "object" and not isinstance(properties, dict):
return walked
closed_properties: Final = properties if isinstance(properties, dict) else dict[str, JsonValue]()
return {
**walked,
"properties": closed_properties,
"required": list(closed_properties),
"additionalProperties": False,
}
def _names(value: object) -> frozenset[str]:
try:
return _NAMES.validate_python(value)

View file

@ -7,19 +7,13 @@ import json
import os
import typing
from collections.abc import Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import Final, TypeAlias
from typing import Final
from pydantic import TypeAdapter, ValidationError
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
if typing.TYPE_CHECKING:
from litellm.harness.context import SessionContext
# A decoded JSON document: what json.loads / model_json_schema() produce.
JSONValue: TypeAlias = "dict[str, JSONValue] | list[JSONValue] | str | int | float | bool | None"
SKILL_MANIFEST: Final = "SKILL.md"
_JSON_DECODER: Final = json.JSONDecoder()
_JSON_OBJECT_ADAPTER: Final = TypeAdapter(dict[str, object])
@ -86,35 +80,6 @@ def turn_prompt(ctx: SessionContext, prompt: str) -> str:
return f"{prompt}\n\n{structured_output_instruction(ctx.output.model_json_schema())}"
def strict_json_schema(schema: JSONValue, depth: int = 0) -> JSONValue:
"""Make a JSON schema acceptable to OpenAI strict structured outputs.
Every object gets `additionalProperties: false` and all of its properties required,
recursively. Keywords strict mode rejects next to $ref are dropped. Nesting deeper than
DEFAULT_MAX_RECURSE_DEPTH raises instead of recursing further.
"""
if depth > DEFAULT_MAX_RECURSE_DEPTH:
raise ValueError(f"output schema is nested deeper than {DEFAULT_MAX_RECURSE_DEPTH} levels")
if isinstance(schema, list):
return [strict_json_schema(entry, depth + 1) for entry in schema] # mutable-ok: JSON document output
if not isinstance(schema, dict):
return schema
entries: Final = ((key, strict_json_schema(value, depth + 1)) for key, value in schema.items())
result = dict(entries) # mutable-ok: JSON document; "default" is popped below
if "$ref" in result:
return {"$ref": result["$ref"]} # mutable-ok: JSONValue output is a plain JSON document
result.pop("default", None)
properties = result.get("properties")
if result.get("type") == "object" or isinstance(properties, dict):
props = properties if isinstance(properties, dict) else {} # mutable-ok: JSONValue object member
required: Final[list[JSONValue]] = list(props) # mutable-ok: JSON array in the output schema
strict: Final[Mapping[str, JSONValue]] = MappingProxyType(
{"properties": props, "required": required, "additionalProperties": False}
)
result = {**result, **strict} # mutable-ok: JSON document output
return result
def decode_json_line(line: bytes | str) -> Mapping[str, object] | None:
"""One JSONL line as a dict, or None for blank / non-JSON / non-object lines."""
text = line.strip()

View file

@ -32,6 +32,7 @@ from litellm.harness.types import (
ToolResult,
)
from litellm.litellm_core_utils.core_helpers import iter_items
from litellm.llms.base_llm.base_utils import strict_json_schema
from litellm.llms.base_llm.harness.transformation import (
BaseCLIHarnessConfig,
HarnessSessionSetup,
@ -40,7 +41,7 @@ from litellm.llms.base_llm.harness.transformation import (
HarnessTurnResponse,
event_list,
)
from litellm.llms.base_llm.harness.utils import stderr_tail_text, strict_json_schema
from litellm.llms.base_llm.harness.utils import stderr_tail_text
if TYPE_CHECKING:
from litellm.harness.context import SessionContext

View file

@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
cache_control_as_prompt_cache_breakpoint,
drop_non_python_regex_patterns,
flatten_combinators_and_drop_non_python_regex_patterns,
prompt_cache_breakpoint_for,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
@ -186,7 +187,7 @@ def _with_prompt_cache_breakpoints_under(item: Mapping[str, object], key: str) -
if item_mark is None:
return item
text_part: Final[Mapping[str, object]] = {"type": "input_text", "text": value}
return {**unmarked_item, key: [with_prompt_cache_breakpoint(text_part, PromptCacheBreakpoint(mode="explicit"))]}
return {**unmarked_item, key: [with_prompt_cache_breakpoint(text_part, prompt_cache_breakpoint_for(item_mark))]}
try:
parts: Final = _CONTENT_PARTS.validate_python(value)
except ValidationError:
@ -212,7 +213,7 @@ def _with_breakpoint_on_last_reading_part(
if not reading_indexes:
return parts
last: Final = reading_indexes[-1]
marked_last: Final = with_prompt_cache_breakpoint(parts[last], PromptCacheBreakpoint(mode="explicit"))
marked_last: Final = with_prompt_cache_breakpoint(parts[last], prompt_cache_breakpoint_for(item_mark))
return (*parts[:last], marked_last, *parts[last + 1 :])

View file

@ -84,6 +84,8 @@ class AnthropicWebSearchTool(TypedDict, total=False):
cache_control: dict | ChatCompletionCachedContent | None
max_uses: int | None
user_location: AnthropicWebSearchUserLocation | None
allowed_domains: ReadOnly[Sequence[str] | None]
blocked_domains: ReadOnly[Sequence[str] | None]
defer_loading: bool | None
allowed_callers: list[str] | None
input_examples: list[dict[str, object]] | None
@ -192,6 +194,7 @@ AllAnthropicToolsValues = (
class AnthropicMcpServerToolConfiguration(TypedDict, total=False):
allowed_tools: list[str] | None
enabled: ReadOnly[bool | None]
class AnthropicMcpServerTool(TypedDict, total=False):
@ -202,6 +205,18 @@ class AnthropicMcpServerTool(TypedDict, total=False):
authorization_token: str
class AnthropicMcpToolConfig(TypedDict, total=False):
enabled: ReadOnly[bool]
defer_loading: ReadOnly[bool]
class AnthropicMcpToolset(TypedDict, total=False):
type: Required[ReadOnly[Literal["mcp_toolset"]]]
mcp_server_name: Required[ReadOnly[str]]
default_config: ReadOnly[AnthropicMcpToolConfig]
configs: ReadOnly[Mapping[str, AnthropicMcpToolConfig]]
class AnthropicMessagesTextParam(TypedDict, total=False):
type: Required[Literal["text"]]
text: Required[str]

View file

@ -65,7 +65,7 @@ IGNORE_FUNCTIONS = [
"_iter_fallback_targets", # max depth set (2 * ROUTER_MAX_FALLBACKS); fails closed by raising ValueError at the cap.
"_mergeable_branch", # max depth set (_MAX_SCHEMA_FLATTEN_DEPTH=32) plus a seen_refs cycle guard; passes the schema through untouched at the cap.
"json_string_leaves", # max depth set (MAX_STRUCTURED_CONTENT_SCAN_DEPTH); fails closed by raising at the cap so nothing goes unscanned.
"strict_json_schema", # harness: max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by raising ValueError at the cap.
"strict_json_schema", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by raising ValueError at the cap.
"toml_value", # harness/codex: max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by raising OptionsMismatch at the cap.
"with_json_string_leaves", # transitively bounded: only runs on a tree json_string_leaves already walked under the cap.
"_clean_extraction", # ScaleDown: traverses the entity map built under MAX_SCHEMA_DEPTH=32 and MAX_ENTITIES=1000.

View file

@ -2328,13 +2328,23 @@ def test_get_semantic_cache_prompt_from_messages_matches_get_str_from_messages_w
assert get_semantic_cache_prompt_from_messages(messages) == get_str_from_messages(messages)
def test_cache_control_is_replaced_by_an_explicit_prompt_cache_breakpoint_on_the_same_block() -> None:
marked: Final = {"type": "text", "text": "rules", "cache_control": {"type": "ephemeral", "ttl": "1h"}}
@pytest.mark.parametrize(
("cache_control", "breakpoint"),
[
pytest.param({"type": "ephemeral", "ttl": "1h"}, {"mode": "explicit", "ttl": "12h"}, id="1h-asks-for-12h"),
pytest.param({"type": "ephemeral", "ttl": "5m"}, {"mode": "explicit"}, id="5m-keeps-the-request-ttl"),
pytest.param({"type": "ephemeral"}, {"mode": "explicit"}, id="no-ttl-keeps-the-request-ttl"),
],
)
def test_cache_control_is_replaced_by_an_explicit_prompt_cache_breakpoint_on_the_same_block(
cache_control: dict[str, str], breakpoint: dict[str, str]
) -> None:
marked: Final = {"type": "text", "text": "rules", "cache_control": cache_control}
assert cache_control_as_prompt_cache_breakpoint(marked) == {
"type": "text",
"text": "rules",
"prompt_cache_breakpoint": {"mode": "explicit"},
"prompt_cache_breakpoint": breakpoint,
}

View file

@ -3869,6 +3869,29 @@ class TestTranslateAnthropicOutputFormatToOpenAI:
assert schema["additionalProperties"] is False
assert sorted(schema["required"]) == ["age", "email", "name"]
def test_keywords_strict_mode_rejects_are_dropped_and_property_names_kept(self):
output_format = {
"type": "json_schema",
"schema": {
"type": "object",
"properties": {
"default": {"type": "string", "default": "x"},
"extra": {"type": "object"},
},
},
}
result = self.adapter.translate_anthropic_output_format_to_openai(output_format)
assert result is not None
assert result["json_schema"]["schema"] == {
"type": "object",
"properties": {
"default": {"type": "string"},
"extra": {"type": "object", "properties": {}, "required": [], "additionalProperties": False},
},
"required": ["default", "extra"],
"additionalProperties": False,
}
def test_invalid_output_format_returns_none(self):
assert (
self.adapter.translate_anthropic_output_format_to_openai("invalid") is None

View file

@ -82,6 +82,54 @@ async def test_openai_messages_forwards_stop_sequences_to_chat_completions(
assert "stop_sequences" not in body
CONTAINER_RESPONSE: Final = {
"id": "resp_container",
"object": "response",
"created_at": 1,
"status": "completed",
"model": "gpt-5.5",
"output": [
{
"type": "message",
"id": "msg_1",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "hi", "annotations": []}],
}
],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
}
@pytest.mark.asyncio
async def test_openai_messages_container_answers_400_unless_drop_params_drops_it(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
request: Final = {
"model": "openai/gpt-5.5",
"max_tokens": 80,
"messages": [{"role": "user", "content": "hi"}],
"container": {"id": "container_abc"},
"api_key": "test-api-key",
}
with respx.mock as upstream:
bridge: Final = upstream.post("https://api.openai.com/v1/responses").mock(
return_value=httpx.Response(200, json=CONTAINER_RESPONSE)
)
with pytest.raises(litellm.BadRequestError) as rejected:
await litellm.anthropic.messages.acreate(**request)
assert rejected.value.status_code == 400
assert "no equivalent for container" in rejected.value.message
assert not bridge.called
await litellm.anthropic.messages.acreate(**request, drop_params=True)
assert "container" not in json.loads(bridge.calls.last.request.content)
@pytest.mark.parametrize("stop_sequences", [None, []], ids=["no-stops", "empty-stops"])
def test_anthropic_messages_handler(stop_sequences):
"""

View file

@ -403,6 +403,9 @@ OTHER_HOST: Final = {"custom_llm_provider": "openai", "api_base": "https://gatew
EPHEMERAL: Final = {"type": "ephemeral"}
LOOKUP_SCHEMA: Final = {"type": "object", "properties": {}}
def _request_with_client_cache_marks(cache_control: dict[str, str] | None) -> dict[str, object]:
mark: Final = {} if cache_control is None else {"cache_control": cache_control}
return {
@ -411,9 +414,18 @@ def _request_with_client_cache_marks(cache_control: dict[str, str] | None) -> di
"messages": [
{"role": "user", "content": [{"type": "text", "text": "look", **mark}, {"type": "text", "text": "now"}]}
],
"tools": [{"name": "lookup", "input_schema": LOOKUP_SCHEMA, **mark}],
}
@pytest.mark.parametrize(
("cache_control", "breakpoint"),
[
pytest.param({**EPHEMERAL, "ttl": "1h"}, {"mode": "explicit", "ttl": "12h"}, id="1h-asks-for-12h"),
pytest.param({**EPHEMERAL, "ttl": "5m"}, {"mode": "explicit"}, id="5m-keeps-the-request-ttl"),
pytest.param(EPHEMERAL, {"mode": "explicit"}, id="no-ttl-keeps-the-request-ttl"),
],
)
@pytest.mark.parametrize(
"extra_kwargs",
[
@ -422,12 +434,12 @@ def _request_with_client_cache_marks(cache_control: dict[str, str] | None) -> di
],
)
def test_client_cache_control_reaches_gpt_5_6_as_a_prompt_cache_breakpoint_on_the_same_block(
extra_kwargs: dict[str, object],
extra_kwargs: dict[str, object], cache_control: dict[str, str], breakpoint: dict[str, str]
) -> None:
responses_kwargs: Final = _build_responses_kwargs(
model="openai/gpt-5.6",
extra_kwargs=extra_kwargs,
**_request_with_client_cache_marks({**EPHEMERAL, "ttl": "1h"}),
**_request_with_client_cache_marks(cache_control),
)
assert "instructions" not in responses_kwargs
@ -437,18 +449,27 @@ def test_client_cache_control_reaches_gpt_5_6_as_a_prompt_cache_breakpoint_on_th
"role": "developer",
"content": [
{"type": "input_text", "text": "Be concise."},
{"type": "input_text", "text": "Be helpful.", "prompt_cache_breakpoint": {"mode": "explicit"}},
{"type": "input_text", "text": "Be helpful.", "prompt_cache_breakpoint": breakpoint},
],
},
{
"type": "message",
"role": "user",
"content": [
{"type": "input_text", "text": "look", "prompt_cache_breakpoint": {"mode": "explicit"}},
{"type": "input_text", "text": "look", "prompt_cache_breakpoint": breakpoint},
{"type": "input_text", "text": "now"},
],
},
]
assert responses_kwargs["tools"] == [
{
"type": "function",
"name": "lookup",
"strict": False,
"parameters": LOOKUP_SCHEMA,
"prompt_cache_breakpoint": breakpoint,
}
]
def test_client_cache_control_on_a_screenshot_inside_a_tool_result_reaches_gpt_5_6_on_that_screenshot() -> None:
@ -499,17 +520,20 @@ RAW_BREAKPOINT: Final = {"mode": "explicit"}
RAW_LOG_OUTPUT: Final = [{"type": "input_text", "text": "build ok", "prompt_cache_breakpoint": RAW_BREAKPOINT}]
RAW_QUESTION: Final = {"type": "input_text", "text": "q", "prompt_cache_breakpoint": RAW_BREAKPOINT}
LOOKUP_FUNCTION: Final = {"type": "function", "name": "lookup", "strict": False, "parameters": LOOKUP_SCHEMA}
RAW_LOOKUP_FUNCTION: Final = {**LOOKUP_FUNCTION, "prompt_cache_breakpoint": RAW_BREAKPOINT}
@pytest.mark.parametrize(
("model", "extra_kwargs", "log_output", "question"),
("model", "extra_kwargs", "log_output", "question", "tool"),
[
pytest.param("openai/gpt-5.5", OPENAI_API, RAW_LOG_OUTPUT, RAW_QUESTION, id="sent-as-is"),
pytest.param("openai/gpt-5.5", OPENAI_API, RAW_LOG_OUTPUT, RAW_QUESTION, RAW_LOOKUP_FUNCTION, id="sent-as-is"),
pytest.param(
"openai/gpt-5.5",
{**OPENAI_API, "drop_params": True},
"build ok",
{"type": "input_text", "text": "q"},
LOOKUP_FUNCTION,
id="dropped-under-drop-params",
),
pytest.param(
@ -517,6 +541,7 @@ RAW_QUESTION: Final = {"type": "input_text", "text": "q", "prompt_cache_breakpoi
{**OTHER_HOST, "drop_params": True},
RAW_LOG_OUTPUT,
RAW_QUESTION,
RAW_LOOKUP_FUNCTION,
id="kept-under-drop-params-for-a-model-that-takes-them",
),
pytest.param(
@ -524,16 +549,18 @@ RAW_QUESTION: Final = {"type": "input_text", "text": "q", "prompt_cache_breakpoi
{**OTHER_HOST, "drop_params": True, "base_model": "gpt-5.6"},
RAW_LOG_OUTPUT,
RAW_QUESTION,
RAW_LOOKUP_FUNCTION,
id="kept-under-drop-params-for-a-base-model-that-takes-them",
),
],
)
def test_raw_prompt_cache_breakpoints_reach_a_deployment_unless_drop_params_drops_them_for_a_model_that_takes_none(
model: str, extra_kwargs: dict[str, object], log_output: object, question: dict[str, object]
model: str, extra_kwargs: dict[str, object], log_output: object, question: dict[str, object], tool: object
) -> None:
responses_kwargs: Final = _build_responses_kwargs(
model=model,
extra_kwargs=extra_kwargs,
tools=[{"name": "lookup", "input_schema": LOOKUP_SCHEMA, "prompt_cache_breakpoint": RAW_BREAKPOINT}],
max_tokens=1024,
messages=[
{
@ -554,6 +581,7 @@ def test_raw_prompt_cache_breakpoints_reach_a_deployment_unless_drop_params_drop
{"type": "function_call_output", "call_id": "toolu_log", "output": log_output},
{"type": "message", "role": "user", "content": [question]},
]
assert responses_kwargs["tools"] == [tool]
@pytest.mark.parametrize(
@ -739,3 +767,165 @@ async def test_recomputed_cost_of_a_served_bridge_response_prices_the_reasoning_
assert logging_obj.cost_breakdown is not None
assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(expected_output)
assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(expected_reasoning)
DOCS_SERVER: Final = {"type": "url", "name": "docs", "url": "https://mcp.example.com/sse"}
@pytest.mark.parametrize(
("speed", "extra_kwargs", "service_tier"),
[
pytest.param("fast", OPENAI_API, "priority", id="fast"),
pytest.param("fast", {**OPENAI_API, "service_tier": "flex"}, "flex", id="forwarded-tier-wins-over-fast"),
pytest.param(None, {**OPENAI_API, "service_tier": "flex"}, "flex", id="forwarded-tier"),
],
)
def test_speed_fast_asks_for_the_priority_tier_unless_a_service_tier_is_forwarded(
speed: str | None, extra_kwargs: dict[str, object], service_tier: str
) -> None:
responses_kwargs: Final = _build_responses_kwargs(
model="openai/gpt-5.6", max_tokens=1024, messages=MESSAGES, speed=speed, extra_kwargs=extra_kwargs
)
assert responses_kwargs["service_tier"] == service_tier
def test_speed_mcp_servers_and_container_never_reach_the_responses_call_raw() -> None:
responses_kwargs: Final = _build_responses_kwargs(
model="openai/gpt-5.6",
max_tokens=1024,
messages=MESSAGES,
speed="fast",
mcp_servers=[DOCS_SERVER],
container={"id": "container_abc"},
extra_kwargs={**OPENAI_API, "drop_params": True},
)
assert {"speed", "mcp_servers", "container"}.isdisjoint(responses_kwargs)
assert responses_kwargs["tools"] == [
{
"type": "mcp",
"server_label": "docs",
"server_url": "https://mcp.example.com/sse",
"require_approval": "never",
}
]
def test_container_answers_400_without_drop_params() -> None:
with pytest.raises(litellm.UnsupportedParamsError) as rejected:
_build_responses_kwargs(
model="openai/gpt-5.6",
max_tokens=1024,
messages=MESSAGES,
container={"id": "container_abc"},
extra_kwargs=OPENAI_API,
)
assert rejected.value.status_code == 400
assert "no equivalent for container" in rejected.value.message
WIRE_RESPONSE: Final = {
"id": "resp_fields",
"object": "response",
"created_at": 1,
"status": "completed",
"model": "gpt-5.6",
"output": [
{
"type": "message",
"id": "msg_1",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "{}", "annotations": []}],
}
],
"parallel_tool_calls": False,
"tool_choice": "auto",
"tools": [],
"usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7},
}
@pytest.mark.parametrize(
("request_fields", "wire_fields"),
[
pytest.param(
{
"tools": [
dict(BASH_TOOL),
{
"type": "web_search_20250305",
"name": "web_search",
"max_uses": 2,
"allowed_domains": ["docs.python.org"],
},
],
"tool_choice": {"type": "auto", "disable_parallel_tool_use": True},
"speed": "fast",
"output_format": {"type": "json_schema", "schema": {"type": "object", "properties": {}}},
},
{
"parallel_tool_calls": False,
"service_tier": "priority",
"max_tool_calls": 2,
"web_search": {"type": "web_search", "filters": {"allowed_domains": ["docs.python.org"]}},
"text": {
"format": {
"type": "json_schema",
"name": "structured_output",
"schema": {"type": "object", "properties": {}, "required": [], "additionalProperties": False},
"strict": True,
}
},
},
id="parallel-speed-web-search-structured-output",
),
pytest.param(
{
"mcp_servers": [{**DOCS_SERVER, "authorization_token": "tok"}],
"tools": [
{
"type": "mcp_toolset",
"mcp_server_name": "docs",
"default_config": {"enabled": False},
"configs": {"search": {"enabled": True}},
}
],
},
{
"mcp": {
"type": "mcp",
"server_label": "docs",
"server_url": "https://mcp.example.com/sse",
"require_approval": "never",
"authorization": "tok",
"allowed_tools": ["search"],
}
},
id="mcp-server",
),
],
)
@pytest.mark.asyncio
async def test_messages_fields_reach_the_openai_responses_request_body(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
request_fields: dict[str, object],
wire_fields: dict[str, object],
) -> None:
monkeypatch.setenv("OPENAI_API_KEY", "sk-fields-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
litellm.in_memory_llm_clients_cache.flush_cache()
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
status_code=200, json=WIRE_RESPONSE
)
await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
max_tokens=1024, messages=MESSAGES, model="openai/gpt-5.6", custom_llm_provider="openai", **request_fields
)
body: Final = json.loads(upstream.calls.last.request.content)
tools_by_type: Final = {tool["type"]: tool for tool in body.get("tools", [])}
sent: Final = {**body, **tools_by_type}
assert {key: sent.get(key) for key in wire_fields} == wire_fields
assert {"speed", "mcp_servers", "output_format"}.isdisjoint(body)

View file

@ -12,6 +12,7 @@ import pytest
from openai.types.responses import ResponseInputImageParam
from pydantic import TypeAdapter
import litellm
from litellm.constants import (
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
@ -138,14 +139,14 @@ class TestOutputConfigStructuredOutput:
}
def test_output_config_format_json_schema_converted(self):
"""output_config.format.json_schema is converted to OpenAI text.format, defaulting strict to False."""
"""output_config.format.json_schema is converted to OpenAI text.format, strict like Anthropic's."""
req = _make_request(output_config={"format": {"type": "json_schema", "schema": self._SCHEMA}})
kwargs = _ADAPTER.translate_request(req)
assert "text" in kwargs
fmt = kwargs["text"]["format"]
assert fmt["type"] == "json_schema"
assert fmt["schema"] == self._SCHEMA
assert fmt["strict"] is False
assert fmt["strict"] is True
assert fmt["name"] == "structured_output"
def test_output_config_format_explicit_strict_true_is_preserved(self):
@ -161,12 +162,12 @@ class TestOutputConfigStructuredOutput:
assert "text" not in kwargs
def test_output_format_still_works(self):
"""The original output_format field still takes precedence when present, defaulting strict to False."""
"""The original output_format field still takes precedence when present, strict like Anthropic's."""
req = _make_request(output_format={"type": "json_schema", "schema": self._SCHEMA})
kwargs = _ADAPTER.translate_request(req)
assert "text" in kwargs
assert kwargs["text"]["format"]["type"] == "json_schema"
assert kwargs["text"]["format"]["strict"] is False
assert kwargs["text"]["format"]["strict"] is True
def test_output_format_explicit_strict_false_is_preserved(self):
"""output_format with an explicit strict=False is preserved as False."""
@ -191,28 +192,101 @@ class TestOutputConfigStructuredOutput:
assert kwargs["text"]["format"]["schema"] == self._SCHEMA
assert kwargs["text"]["format"]["strict"] is False
def test_optional_property_stays_out_of_required_list(self):
"""A property absent from required must stay absent from required in the translated schema."""
schema = {
_OPTIONAL_NICKNAME: Final = {
"type": "object",
"properties": {
"name": {"type": "string"},
"nickname": {"type": "string", "default": "none"},
"address": {"type": "object", "properties": {"city": {"type": "string"}}},
},
"required": ["name"],
}
_NOT_JSON: Final = {"type": "object", "properties": {"size": {"enum": ("s", "m")}}}
@pytest.mark.parametrize(
("fields", "expected_format"),
[
pytest.param(
{"output_config": {"format": {"type": "json_schema", "schema": _OPTIONAL_NICKNAME}}},
{
"type": "json_schema",
"name": "structured_output",
"schema": {
"type": "object",
"properties": {
"name": {"type": "string"},
"nickname": {"type": "string"},
"address": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
"additionalProperties": False,
},
},
"required": ["name", "nickname", "address"],
"additionalProperties": False,
},
"strict": True,
},
id="unset-strict-closes-every-object",
),
pytest.param(
{"output_format": {"type": "json_schema", "schema": _OPTIONAL_NICKNAME, "strict": False}},
{"type": "json_schema", "name": "structured_output", "schema": _OPTIONAL_NICKNAME, "strict": False},
id="explicit-strict-false-keeps-the-schema",
),
pytest.param(
{"output_format": {"type": "json_schema", "schema": _NOT_JSON}},
{"type": "json_schema", "name": "structured_output", "schema": _NOT_JSON, "strict": False},
id="schema-that-is-not-json-goes-up-as-is",
),
pytest.param({"output_format": {"type": "text"}}, None, id="not-json-schema"),
pytest.param({"output_format": {"type": "json_schema", "schema": {}}}, None, id="empty-schema"),
],
)
def test_json_schema_output_goes_up_strict_unless_the_caller_or_the_schema_rules_it_out(
self, fields: Dict[str, object], expected_format: Dict[str, object] | None
):
"""Strict mode requires every property, so an optional one becomes required. Output that fills it still
matches the caller's schema, which a nullable optional would not."""
kwargs = _ADAPTER.translate_request(_make_request(**fields))
assert kwargs.get("text", {}).get("format") == expected_format
def test_strict_schema_keeps_properties_named_like_schema_keywords(self):
schema: Final = {
"type": "object",
"properties": {
"name": {"type": "string"},
"nickname": {"type": "string"},
"default": {"type": "string"},
"$ref": {"type": "string"},
"properties": {"type": "object", "properties": {"x": {"type": "integer"}}},
"mode": {"enum": [{"type": "object"}, "fast"]},
},
"required": ["name"],
"additionalProperties": False,
}
req = _make_request(output_format={"type": "json_schema", "schema": schema})
kwargs = _ADAPTER.translate_request(req)
fmt_schema = kwargs["text"]["format"]["schema"]
assert fmt_schema["required"] == ["name"]
assert "nickname" not in fmt_schema["required"]
assert fmt_schema["additionalProperties"] is False
assert kwargs["text"]["format"]["schema"] == {
"type": "object",
"properties": {
"default": {"type": "string"},
"$ref": {"type": "string"},
"properties": {
"type": "object",
"properties": {"x": {"type": "integer"}},
"required": ["x"],
"additionalProperties": False,
},
"mode": {"enum": [{"type": "object"}, "fast"]},
},
"required": ["default", "$ref", "properties", "mode"],
"additionalProperties": False,
}
def test_translate_request_does_not_mutate_input_schema(self):
@pytest.mark.parametrize("strict", [pytest.param({}, id="strict"), pytest.param({"strict": False}, id="loose")])
def test_translate_request_does_not_mutate_input_schema(self, strict: Dict[str, bool]):
"""translate_request must not mutate the caller's output_format or schema dicts."""
schema = {"type": "object", "properties": {"x": {"type": "number"}}, "required": ["x"]}
output_format = {"type": "json_schema", "schema": schema, "strict": False}
schema = {"type": "object", "properties": {"x": {"type": "number", "default": 1}}, "required": []}
output_format = {"type": "json_schema", "schema": schema, **strict}
req = _make_request(output_format=output_format)
snapshot = json.loads(json.dumps(output_format))
@ -2424,3 +2498,277 @@ class TestPromptCacheBreakpointToResponses:
extra_kwargs={"prompt_cache_options": {"mode": "explicit"}},
)
assert kwargs["prompt_cache_options"] == {"mode": "explicit"}
# ---------------------------------------------------------------------------
# Anthropic fields with a Responses equivalent map to it, the rest answer 400 unless drop_params drops them
# ---------------------------------------------------------------------------
LOOKUP_TOOL: Final = {"name": "lookup", "input_schema": {"type": "object", "properties": {}}}
DOCS_SERVER: Final = {"type": "url", "name": "docs", "url": "https://mcp.example.com/sse"}
DOCS_MCP_TOOL: Final = {
"type": "mcp",
"server_label": "docs",
"server_url": "https://mcp.example.com/sse",
"require_approval": "never",
}
SF_LOCATION: Final = {"type": "approximate", "city": "San Francisco", "country": "US"}
def _translate(drop_params: bool, **fields: object) -> Dict[str, Any]:
return _ADAPTER.translate_request(_make_request(**fields), drop_params=drop_params)
def _unsupported_message(**fields: object) -> str:
with pytest.raises(litellm.UnsupportedParamsError) as rejected:
_translate(False, **fields)
assert rejected.value.status_code == 400
return rejected.value.message
@pytest.mark.parametrize(
("tool_choice", "expected"),
[
pytest.param({"type": "auto", "disable_parallel_tool_use": True}, {"parallel_tool_calls": False}, id="auto"),
pytest.param({"type": "any", "disable_parallel_tool_use": True}, {"parallel_tool_calls": False}, id="any"),
pytest.param(
{"type": "tool", "name": "lookup", "disable_parallel_tool_use": True},
{"parallel_tool_calls": False},
id="tool",
),
pytest.param({"type": "none", "disable_parallel_tool_use": True}, {}, id="none-calls-no-tools"),
pytest.param({"type": "auto", "disable_parallel_tool_use": False}, {}, id="parallel-allowed"),
pytest.param({"type": "auto"}, {}, id="unset"),
],
)
def test_disable_parallel_tool_use_turns_off_parallel_tool_calls(
tool_choice: Dict[str, object], expected: Dict[str, bool]
):
kwargs: Final = _translate(False, tools=[LOOKUP_TOOL], tool_choice=tool_choice)
assert {key: kwargs[key] for key in ("parallel_tool_calls",) if key in kwargs} == expected
@pytest.mark.parametrize(
("speed", "expected"),
[
pytest.param("fast", {"service_tier": "priority"}, id="fast"),
pytest.param("standard", {}, id="standard"),
pytest.param(None, {}, id="null"),
],
)
def test_speed_maps_to_the_priority_service_tier(speed: object, expected: Dict[str, str]):
kwargs: Final = _translate(False, speed=speed)
assert {key: kwargs[key] for key in ("service_tier", "speed") if key in kwargs} == expected
def test_an_unknown_speed_answers_400_unless_drop_params_drops_it():
assert "speed 'turbo'" in _unsupported_message(speed="turbo")
dropped: Final = _translate(True, speed="turbo")
assert "service_tier" not in dropped and "speed" not in dropped
@pytest.mark.parametrize(
("server", "expected_tools"),
[
pytest.param(DOCS_SERVER, [DOCS_MCP_TOOL], id="every-tool"),
pytest.param(
{**DOCS_SERVER, "authorization_token": "tok"},
[{**DOCS_MCP_TOOL, "authorization": "tok"}],
id="authorization-token",
),
pytest.param(
{**DOCS_SERVER, "tool_configuration": {"allowed_tools": ["search", "fetch"]}},
[{**DOCS_MCP_TOOL, "allowed_tools": ["search", "fetch"]}],
id="deprecated-allowed-tools",
),
pytest.param(
{**DOCS_SERVER, "tool_configuration": {"enabled": True}}, [DOCS_MCP_TOOL], id="deprecated-enabled"
),
pytest.param({**DOCS_SERVER, "tool_configuration": {"enabled": False}}, [], id="deprecated-disabled"),
],
)
def test_an_mcp_server_becomes_a_responses_mcp_tool(server: Dict[str, object], expected_tools: List[object]):
kwargs: Final = _translate(False, mcp_servers=[server])
assert kwargs.get("tools", []) == expected_tools
assert "mcp_servers" not in kwargs
@pytest.mark.parametrize(
("toolset", "expected_tools"),
[
pytest.param({}, [DOCS_MCP_TOOL], id="defaults"),
pytest.param(
{"default_config": {"enabled": True}, "configs": {"search": {"enabled": True}}},
[DOCS_MCP_TOOL],
id="enabled-nothing-disabled",
),
pytest.param(
{
"default_config": {"enabled": False},
"configs": {"search": {"enabled": True}, "fetch": {"enabled": False}, "list": {}},
},
[{**DOCS_MCP_TOOL, "allowed_tools": ["search"]}],
id="allowlist",
),
pytest.param({"default_config": {"enabled": False}}, [], id="everything-disabled"),
pytest.param(
{"default_config": {"defer_loading": False}, "configs": {"search": {"defer_loading": False}}},
[DOCS_MCP_TOOL],
id="defer-loading-off",
),
],
)
def test_an_mcp_toolset_sets_the_mcp_tools_allowlist_and_never_becomes_a_function_tool(
toolset: Dict[str, object], expected_tools: List[object]
):
kwargs: Final = _translate(
False,
mcp_servers=[{**DOCS_SERVER, "tool_configuration": {"allowed_tools": ["ignored"]}}],
tools=[LOOKUP_TOOL, {"type": "mcp_toolset", "mcp_server_name": "docs", **toolset}],
)
assert kwargs["tools"] == [_ADAPTER.translate_tools_to_responses_api([LOOKUP_TOOL])[0], *expected_tools]
def test_an_mcp_toolset_for_another_server_is_ignored():
kwargs: Final = _translate(
False,
mcp_servers=[DOCS_SERVER],
tools=[{"type": "mcp_toolset", "mcp_server_name": "other", "default_config": {"enabled": False}}],
)
assert kwargs["tools"] == [DOCS_MCP_TOOL]
UNMAPPABLE_TOOLSETS: Final = [
pytest.param(
{"configs": {"delete": {"enabled": False}, "drop": {"enabled": False}}},
"mcp_toolset 'docs' disabling delete, drop while its other tools stay enabled",
[],
id="denylist",
),
pytest.param(
{"default_config": {"defer_loading": True}},
"defer_loading on mcp_toolset 'docs'",
[DOCS_MCP_TOOL],
id="default-defer-loading",
),
pytest.param(
{"default_config": {"enabled": False}, "configs": {"search": {"enabled": True, "defer_loading": True}}},
"defer_loading on mcp_toolset 'docs'",
[{**DOCS_MCP_TOOL, "allowed_tools": ["search"]}],
id="tool-defer-loading",
),
pytest.param({"configs": ["search"]}, "the malformed mcp_toolset for 'docs'", [], id="malformed"),
]
@pytest.mark.parametrize(("toolset", "field", "dropped_to"), UNMAPPABLE_TOOLSETS)
def test_an_mcp_toolset_without_a_responses_equivalent_answers_400_naming_it(
toolset: Dict[str, object], field: str, dropped_to: List[object]
):
message: Final = _unsupported_message(
mcp_servers=[DOCS_SERVER], tools=[{"type": "mcp_toolset", "mcp_server_name": "docs", **toolset}]
)
assert field in message
@pytest.mark.parametrize(("toolset", "field", "dropped_to"), UNMAPPABLE_TOOLSETS)
def test_drop_params_drops_what_an_mcp_toolset_cannot_map(
toolset: Dict[str, object], field: str, dropped_to: List[object]
):
kwargs: Final = _translate(
True, mcp_servers=[DOCS_SERVER], tools=[{"type": "mcp_toolset", "mcp_server_name": "docs", **toolset}]
)
assert kwargs.get("tools", []) == dropped_to
@pytest.mark.parametrize(
("web_search", "expected"),
[
pytest.param({}, {"type": "web_search_preview"}, id="no-filters"),
pytest.param(
{"allowed_domains": ["docs.python.org"]},
{"type": "web_search", "filters": {"allowed_domains": ["docs.python.org"]}},
id="allowed-domains",
),
pytest.param(
{"blocked_domains": ["example.com"], "allowed_domains": None, "user_location": SF_LOCATION},
{"type": "web_search", "filters": {"blocked_domains": ["example.com"]}, "user_location": SF_LOCATION},
id="blocked-domains-and-location",
),
pytest.param(
{"user_location": SF_LOCATION},
{"type": "web_search_preview", "user_location": SF_LOCATION},
id="location-only",
),
],
)
def test_web_search_keeps_its_domain_filters_and_user_location(web_search: Dict[str, object], expected: object):
kwargs: Final = _translate(
False, tools=[{"type": "web_search_20250305", "name": "web_search", "max_uses": 3, **web_search}]
)
assert kwargs["tools"] == [expected]
assert kwargs["max_tool_calls"] == 3
@pytest.mark.parametrize(
("max_uses", "expected"),
[
pytest.param({"max_uses": 2}, {"max_tool_calls": 2}, id="capped"),
pytest.param({}, {}, id="no-cap"),
],
)
def test_web_search_max_uses_caps_tool_calls_next_to_function_tools(
max_uses: Dict[str, int], expected: Dict[str, int]
):
kwargs: Final = _translate(
False, tools=[LOOKUP_TOOL, {"type": "web_search_20250305", "name": "web_search", **max_uses}]
)
assert {key: kwargs[key] for key in ("max_tool_calls",) if key in kwargs} == expected
def test_web_search_max_uses_next_to_another_built_in_tool_answers_400_naming_it():
message: Final = _unsupported_message(
mcp_servers=[DOCS_SERVER], tools=[{"type": "web_search_20250305", "name": "web_search", "max_uses": 2}]
)
assert "max_uses on tool 'web_search' next to another built-in tool" in message
def test_drop_params_drops_a_web_search_max_uses_it_cannot_map():
kwargs: Final = _translate(
True, mcp_servers=[DOCS_SERVER], tools=[{"type": "web_search_20250305", "name": "web_search", "max_uses": 2}]
)
assert kwargs["tools"] == [{"type": "web_search_preview"}, DOCS_MCP_TOOL]
assert "max_tool_calls" not in kwargs
@pytest.mark.parametrize("allowed_callers", [["code_execution_20250825"], ["direct", "code_execution_20250825"]])
def test_allowed_callers_answers_400_unless_drop_params_drops_it(allowed_callers: List[str]):
tools: Final = [{**LOOKUP_TOOL, "allowed_callers": allowed_callers}]
assert "allowed_callers on tool 'lookup'" in _unsupported_message(tools=tools)
assert _translate(True, tools=tools)["tools"] == _ADAPTER.translate_tools_to_responses_api([LOOKUP_TOOL])
def test_container_answers_400_unless_drop_params_drops_it():
message: Final = _unsupported_message(container={"id": "container_abc"})
assert "no equivalent for container. To drop it, set `litellm.drop_params=True`" in message
assert "container" not in _translate(True, container={"id": "container_abc"})
def test_one_400_names_every_unsupported_field_and_how_to_drop_them():
direct_only_tool: Final = {**LOOKUP_TOOL, "name": "fetch", "allowed_callers": ["direct"]}
message: Final = _unsupported_message(
speed="turbo",
container={"id": "container_abc"},
tools=[{**LOOKUP_TOOL, "allowed_callers": ["x"]}, direct_only_tool],
)
assert message.startswith(
"litellm.UnsupportedParamsError: openai.gpt-5.1-codex is served through the OpenAI Responses API, which has no "
"equivalent for allowed_callers on tool 'lookup'; speed 'turbo'; container. "
)
assert "To drop them, set `litellm.drop_params=True`" in message
def test_a_function_tool_keeps_its_prompt_cache_breakpoint():
breakpoint: Final = {"mode": "explicit", "ttl": "12h"}
kwargs: Final = _translate(False, tools=[{**LOOKUP_TOOL, "prompt_cache_breakpoint": breakpoint}])
assert kwargs["tools"][0]["prompt_cache_breakpoint"] == breakpoint

View file

@ -5,14 +5,16 @@ codex-cli through a LiteLLM gateway.
"""
import asyncio
import functools
import json
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional
import pytest
from pydantic import BaseModel, ValidationError
from pydantic import BaseModel, JsonValue, ValidationError
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.harness.context import SessionContext
from litellm.harness.errors import HarnessError, HarnessInstallFailed, OptionsMismatch
from litellm.harness.handlers.cli_handler import CLIHarnessHandler
@ -20,11 +22,11 @@ from litellm.harness.options import ClaudeCodeOptions, CodexOptions
from litellm.harness.sandbox.base import CompletedRun
from litellm.harness.sandbox.docker import DockerSandbox
from litellm.harness.types import Harness, Reasoning, Text, ToolCall, ToolResult
from litellm.llms.base_llm.base_utils import strict_json_schema
from litellm.llms.base_llm.harness.transformation import (
HarnessTurnError,
HarnessTurnRequest,
)
from litellm.llms.base_llm.harness.utils import strict_json_schema
from litellm.llms.codex.harness.transformation import (
CODEX_SCHEMA_FILENAME,
CODEX_TOKEN_ENV,
@ -343,6 +345,61 @@ def test_strict_json_schema_recursive():
assert answer["required"] == ["file", "content"]
def test_strict_json_schema_treats_property_names_and_enum_values_as_data():
schema = strict_json_schema(
{
"type": "object",
"properties": {
"default": {"type": "string"},
"$ref": {"type": "string", "default": "x"},
"properties": {"type": "object", "properties": {"x": {"type": "integer"}}},
"kind": {"enum": [{"type": "object"}], "const": {"type": "object"}, "examples": [{"type": "object"}]},
"untyped": {"properties": {"z": {"type": "string"}}},
},
"$defs": {"default": {"type": "object", "properties": {"y": {"type": "string"}}}},
"definitions": {"default": {"type": "string"}},
"patternProperties": {"default": {"type": "string"}},
}
)
assert schema == {
"type": "object",
"properties": {
"default": {"type": "string"},
"$ref": {"type": "string"},
"properties": {
"type": "object",
"properties": {"x": {"type": "integer"}},
"required": ["x"],
"additionalProperties": False,
},
"kind": {"enum": [{"type": "object"}], "const": {"type": "object"}, "examples": [{"type": "object"}]},
"untyped": {"properties": {"z": {"type": "string"}}, "required": ["z"], "additionalProperties": False},
},
"$defs": {
"default": {
"type": "object",
"properties": {"y": {"type": "string"}},
"required": ["y"],
"additionalProperties": False,
}
},
"definitions": {"default": {"type": "string"}},
"patternProperties": {"default": {"type": "string"}},
"required": ["default", "$ref", "properties", "kind", "untyped"],
"additionalProperties": False,
}
def _nested_items(levels: int) -> JsonValue:
return functools.reduce(lambda inner, _: {"items": inner}, range(levels), True)
def test_strict_json_schema_refuses_a_schema_nested_past_the_recursion_cap():
assert strict_json_schema(_nested_items(DEFAULT_MAX_RECURSE_DEPTH)) == _nested_items(DEFAULT_MAX_RECURSE_DEPTH)
with pytest.raises(ValueError, match="nested deeper than"):
strict_json_schema(_nested_items(DEFAULT_MAX_RECURSE_DEPTH + 1))
def test_config_overrides_rejects_managed_keys():
for key in (
"model_provider",

View file

@ -945,6 +945,7 @@ class TestCacheControlPreservationForCustomEndpoint:
_EPHEMERAL: Final[dict[str, str]] = {"type": "ephemeral"}
_EXPLICIT: Final[dict[str, str]] = {"mode": "explicit"}
_EXPLICIT_12H: Final[dict[str, str]] = {"mode": "explicit", "ttl": "12h"}
_LOOKUP_TOOL: Final[dict[str, object]] = {"type": "function", "function": {"name": "lookup", "parameters": {}}}
@ -956,8 +957,8 @@ def _marked(part: dict[str, object], mark: dict[str, str] = _EPHEMERAL) -> dict[
return {**part, "cache_control": mark}
def _breakpoint(part: dict[str, object]) -> dict[str, object]:
return {**part, "prompt_cache_breakpoint": _EXPLICIT}
def _breakpoint(part: dict[str, object], breakpoint: dict[str, str] = _EXPLICIT) -> dict[str, object]:
return {**part, "prompt_cache_breakpoint": breakpoint}
class TestCacheControlAsPromptCacheBreakpoint:
@ -1000,7 +1001,7 @@ class TestCacheControlAsPromptCacheBreakpoint:
{
"role": "user",
"content": [
_breakpoint(_text("Describe this")),
_breakpoint(_text("Describe this"), _EXPLICIT_12H),
_breakpoint({"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}),
_text("and nothing else"),
],
@ -1050,7 +1051,7 @@ class TestCacheControlAsPromptCacheBreakpoint:
body = self._transform("gpt-5.6", messages)
assert body["messages"] == [
{"role": "system", "content": [_breakpoint(_text("You are helpful."))]},
{"role": "user", "content": [_breakpoint(_text("Hello"))]},
{"role": "user", "content": [_breakpoint(_text("Hello"), _EXPLICIT_12H)]},
{"role": "user", "content": "unmarked"},
]

View file

@ -2866,6 +2866,14 @@ class TestCacheControlBecomesPromptCacheBreakpoint:
headers={},
)
@pytest.mark.parametrize(
("cache_control", "breakpoint"),
[
pytest.param({**_EPHEMERAL, "ttl": "1h"}, {**_EXPLICIT_BREAKPOINT, "ttl": "12h"}, id="1h-asks-for-12h"),
pytest.param({**_EPHEMERAL, "ttl": "5m"}, _EXPLICIT_BREAKPOINT, id="5m-keeps-the-request-ttl"),
pytest.param(_EPHEMERAL, _EXPLICIT_BREAKPOINT, id="no-ttl-keeps-the-request-ttl"),
],
)
@pytest.mark.parametrize(
("litellm_params", "extra_params"),
[
@ -2878,29 +2886,33 @@ class TestCacheControlBecomesPromptCacheBreakpoint:
],
)
def test_marks_reach_gpt_5_6_as_explicit_breakpoints_on_the_same_blocks(
self, litellm_params: GenericLiteLLMParams, extra_params: dict[str, object]
self,
litellm_params: GenericLiteLLMParams,
extra_params: dict[str, object],
cache_control: dict[str, str],
breakpoint: dict[str, str],
) -> None:
result = self._transform("gpt-5.6", litellm_params, {**_EPHEMERAL, "ttl": "1h"}, extra_params)
result = self._transform("gpt-5.6", litellm_params, cache_control, extra_params)
assert result["input"] == [
{
"role": "developer",
"content": [
{"type": "input_text", "text": "Be concise."},
{"type": "input_text", "text": "Be helpful.", "prompt_cache_breakpoint": _EXPLICIT_BREAKPOINT},
{"type": "input_text", "text": "Be helpful.", "prompt_cache_breakpoint": breakpoint},
],
},
{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "look", "prompt_cache_breakpoint": _EXPLICIT_BREAKPOINT}],
"content": [{"type": "input_text", "text": "look", "prompt_cache_breakpoint": breakpoint}],
},
{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "looking"}]},
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
{
"type": "function_call_output",
"call_id": "call_1",
"output": [{"type": "input_text", "text": "found", "prompt_cache_breakpoint": _EXPLICIT_BREAKPOINT}],
"output": [{"type": "input_text", "text": "found", "prompt_cache_breakpoint": breakpoint}],
},
{"type": "function_call", "call_id": "call_2", "name": "lookup", "arguments": "{}"},
{
@ -2908,7 +2920,7 @@ class TestCacheControlBecomesPromptCacheBreakpoint:
"call_id": "call_2",
"output": [
{"type": "input_text", "text": "first"},
{"type": "input_text", "text": "last", "prompt_cache_breakpoint": _EXPLICIT_BREAKPOINT},
{"type": "input_text", "text": "last", "prompt_cache_breakpoint": breakpoint},
],
},
]