scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

Merge 240ed797bd into f82e4c80ac

Browse source
This commit is contained in:
Matthew Lapointe 2026-11-06 05:36:03 +00:00 • committed by GitHub
commit 7e174d7083
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 1586 additions and 123 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
litellm/completion_extras/litellm_responses_transformation/handler.py
View file

@ -11,6 +11,7 @@ from typing import (
cast, # noqa: TID251 # optional_params reaches the bridge as an untyped dict
)
import httpx
from typing_extensions import ReadOnly, TypedDict
from litellm.types.llms.openai import ResponsesAPIResponse
@ -29,6 +30,17 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict):
logging_obj: ReadOnly["LiteLLMLoggingObj"]
custom_llm_provider: ReadOnly[str]
encoding: ReadOnly[object]
client: ReadOnly[object]
def _provider_response_headers(stream: object) -> httpx.Headers | None:
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
match stream:
case BaseResponsesAPIStreamingIterator(response=httpx.Response(headers=headers)):
return headers
case _:
return None
def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:
@ -158,6 +170,7 @@ class ResponsesToCompletionBridgeHandler:
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
encoding=typed_kwargs.get("encoding"),
client=typed_kwargs.get("client"),
)
def completion(
@ -193,7 +206,7 @@ class ResponsesToCompletionBridgeHandler:
litellm_params=litellm_params,
headers=headers,
litellm_logging_obj=logging_obj,
client=kwargs.get("client"),
client=validated_kwargs["client"],
)
# Set on request_data rather than passed as explicit kwargs: the spread of
@ -252,7 +265,7 @@ class ResponsesToCompletionBridgeHandler:
else:
if self._is_preformatted_cached_chat_stream(result):
return self._apply_post_stream_processing(result, model, custom_llm_provider)
from .transformation import is_structured_output_request
from .transformation import is_structured_output_request, reasoning_origin_for
completion_stream: Final = self.transformation_handler.get_model_response_iterator(
streaming_response=result,
@ -261,6 +274,7 @@ class ResponsesToCompletionBridgeHandler:
structured_output=is_structured_output_request(
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
),
reasoning_origin=reasoning_origin_for(custom_llm_provider, model),
replays_openai_history=custom_llm_provider == "openai",
)
streamwrapper: Final = CustomStreamWrapper(
@ -268,6 +282,7 @@ class ResponsesToCompletionBridgeHandler:
model=model,
custom_llm_provider=custom_llm_provider,
logging_obj=logging_obj,
_response_headers=_provider_response_headers(result),
)
return self._apply_post_stream_processing(streamwrapper, model, custom_llm_provider)
@ -296,6 +311,7 @@ class ResponsesToCompletionBridgeHandler:
litellm_params=litellm_params,
headers=headers,
litellm_logging_obj=logging_obj,
client=validated_kwargs["client"],
)
except Exception as e:
raise e
@ -357,7 +373,7 @@ class ResponsesToCompletionBridgeHandler:
else:
if self._is_preformatted_cached_chat_stream(result):
return self._apply_post_stream_processing(result, model, custom_llm_provider)
from .transformation import is_structured_output_request
from .transformation import is_structured_output_request, reasoning_origin_for
completion_stream: Final = self.transformation_handler.get_model_response_iterator(
streaming_response=result,
@ -366,6 +382,7 @@ class ResponsesToCompletionBridgeHandler:
structured_output=is_structured_output_request(
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
),
reasoning_origin=reasoning_origin_for(custom_llm_provider, model),
replays_openai_history=custom_llm_provider == "openai",
)
streamwrapper: Final = CustomStreamWrapper(
@ -373,6 +390,7 @@ class ResponsesToCompletionBridgeHandler:
model=model,
custom_llm_provider=custom_llm_provider,
logging_obj=logging_obj,
_response_headers=_provider_response_headers(result),
)
return self._apply_post_stream_processing(streamwrapper, model, custom_llm_provider)

119
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -4,6 +4,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req
import json
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from itertools import accumulate, chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, Union, cast
@ -343,6 +344,81 @@ def _reasoning_items_from_output_items(output_items: Sequence[object]) -> tuple[
)
_REASONING_ORIGIN_PREFIX: Final = "litellm_reasoning_origin:"
@dataclass(frozen=True, slots=True)
class ReasoningReplayTarget:
origin: str | None
def reasoning_origin_for(provider: str | None, model: str) -> str | None:
if provider != "bedrock":
return None
from litellm.llms.bedrock.common_utils import get_bedrock_base_model
return f"{_REASONING_ORIGIN_PREFIX}bedrock/{get_bedrock_base_model(model)}:"
def _tool_reference_as_text(part: object) -> object:
match part:
case {"type": "tool_reference", "tool_name": str() as tool_name}:
return {"type": "text", "text": tool_name}
case _:
return part
def _tool_message_with_tool_references_as_text(msg: Mapping[str, object]) -> Mapping[str, object]:
content: Final = msg.get("content")
if msg.get("role") != "tool" or not isinstance(content, list):
return msg
parts: Final = cast("list[object]", content) # cast-ok: isinstance leaves the part type unknown
return {**msg, "content": [_tool_reference_as_text(part) for part in parts]}
def _tool_references_as_text(messages: Sequence["AllMessageValues"]) -> tuple["AllMessageValues", ...]:
return cast( # cast-ok: each message keeps its fields, and only tool content parts change
'tuple["AllMessageValues", ...]', tuple(_tool_message_with_tool_references_as_text(msg) for msg in messages)
)
def _with_reasoning_origin(output_items: Sequence[object], origin: str | None) -> Sequence[object]:
if origin is None:
return output_items
return tuple(_reasoning_output_item_with_origin(item, origin) for item in output_items)
def _reasoning_output_item_with_origin(item: object, origin: str) -> object:
from openai.types.responses import ResponseReasoningItem
if isinstance(item, ResponseReasoningItem):
return (
item.model_copy(update={"encrypted_content": f"{origin}{item.encrypted_content}"})
if item.encrypted_content
else item
)
if not isinstance(item, dict):
return item
fields: Final = cast(dict[str, object], item) # cast-ok: provider JSON item, keys forwarded verbatim
encrypted_content: Final = fields.get("encrypted_content")
if fields.get("type") != "reasoning" or not isinstance(encrypted_content, str) or not encrypted_content:
return fields
return {**fields, "encrypted_content": f"{origin}{encrypted_content}"}
def _replayable_reasoning_item(
item: Mapping[str, object], target: ReasoningReplayTarget
) -> Mapping[str, object] | None:
encrypted_content: Final = item.get("encrypted_content")
if not isinstance(encrypted_content, str) or not encrypted_content:
return item
if target.origin is None:
return None if encrypted_content.startswith(_REASONING_ORIGIN_PREFIX) else item
if not encrypted_content.startswith(target.origin):
return None
return {**item, "encrypted_content": encrypted_content.removeprefix(target.origin)}
_TOOL_SEARCH_ITEM_TYPES: Final = frozenset({"tool_search_call", "tool_search_output"})
_TOOL_SEARCH_REPLAY_ITEM_TYPES: Final = frozenset({"reasoning", *_TOOL_SEARCH_ITEM_TYPES})
_TOOL_SEARCH_ITEMS_KEY: Final = "tool_search_items"
@ -466,11 +542,18 @@ def _reasoning_ids(items: Sequence[Mapping[str, object]]) -> frozenset[str]:
return frozenset(item_id for item_id in ids if isinstance(item_id, str))
def _assistant_replay_items(msg: "AllMessageValues", replay_tool_search: bool) -> tuple[Mapping[str, object], ...]:
def _assistant_replay_items(
msg: "AllMessageValues", replay_tool_search: bool, replay_to: ReasoningReplayTarget | None
) -> tuple[Mapping[str, object], ...]:
searched: Final = _tool_search_input_items(msg) if replay_tool_search else ()
searched_reasoning_ids: Final = _reasoning_ids(searched)
reasoning: Final = (item for item in _reasoning_input_items(msg) if item.get("id") not in searched_reasoning_ids)
return (*searched, *reasoning)
replayable: Final = (
reasoning
if replay_to is None
else (kept for item in reasoning if (kept := _replayable_reasoning_item(item, replay_to)) is not None)
)
return (*searched, *replayable)
def _as_chat_reasoning_items(
@ -871,13 +954,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
def convert_chat_completion_messages_to_responses_api(
self,
messages: list["AllMessageValues"],
messages: Sequence["AllMessageValues"],
*,
drop_params: bool = False,
target_trims_breakpoints: bool = True,
keep_prompt_cache_breakpoints: bool = False,
replay_tool_search: bool = False,
include_namespace: bool = False,
replay_to: ReasoningReplayTarget | None = None,
replay_openai_history: bool = False,
) -> tuple[list[object], str | None]:
converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input(
@ -886,6 +970,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
target_trims_breakpoints=target_trims_breakpoints,
replay_tool_search=replay_tool_search,
include_namespace=include_namespace,
replay_to=replay_to,
replay_openai_history=replay_openai_history,
)
return (
@ -897,12 +982,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
def _convert_chat_completion_messages_to_responses_input(
self,
messages: list["AllMessageValues"],
messages: Sequence["AllMessageValues"],
*,
drop_params: bool = False,
target_trims_breakpoints: bool = True,
replay_tool_search: bool = False,
include_namespace: bool = False,
replay_to: ReasoningReplayTarget | None = None,
replay_openai_history: bool = False,
) -> tuple[list[object], str | None]:
input_items: Final[list[object]] = []
@ -996,7 +1082,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
)
elif role == "assistant" and tool_calls and isinstance(tool_calls, list):
input_items.extend(_assistant_replay_items(msg, replay_tool_search))
input_items.extend(_assistant_replay_items(msg, replay_tool_search, replay_to))
if content:
input_items.append(
{
@ -1041,7 +1127,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
raise ValueError(f"tool call not supported: {tool_call}")
elif content is not None:
if role == "assistant":
input_items.extend(_assistant_replay_items(msg, replay_tool_search))
input_items.extend(_assistant_replay_items(msg, replay_tool_search, replay_to))
input_items.append(
{
"type": "message",
@ -1056,7 +1142,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
}
)
elif role == "assistant":
input_items.extend(_assistant_replay_items(msg, replay_tool_search))
input_items.extend(_assistant_replay_items(msg, replay_tool_search, replay_to))
return input_items, instructions
@ -1174,12 +1260,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
supports_openai_prompt_cache_breakpoint(name, provider, model_info) for name in named_models
)
converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api(
messages,
_tool_references_as_text(messages) if provider == "bedrock" else messages,
drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
target_trims_breakpoints=drops_unsupported_params(litellm_params.get("drop_params")),
keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast
include_namespace=True,
replay_to=ReasoningReplayTarget(origin=reasoning_origin_for(provider, model)),
replay_openai_history=provider == "openai",
)
# Keep the system text in input instead of instructions when the Responses API needs it there.
@ -1547,8 +1634,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
# Convert response output to choices using the static helper
tagged_output_items: Final = _with_reasoning_origin(
output_items, reasoning_origin_for(_served_provider(litellm_params), model)
)
choices: Final = self._convert_response_output_to_choices(
output_items=output_items,
output_items=tagged_output_items,
handle_raw_dict_callback=self._handle_raw_dict_response_item,
structured_output=is_structured_output_request(
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
@ -1572,7 +1662,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
raw_response.incomplete_details.reason if raw_response.incomplete_details is not None else None
)
if len(choices) == 0:
choices.append(self._build_empty_choice(output_items, incomplete_finish_reason))
choices.append(self._build_empty_choice(tagged_output_items, incomplete_finish_reason))
else:
for choice in choices:
choice.finish_reason = incomplete_finish_reason
@ -1621,6 +1711,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
sync_stream: bool,
json_mode: bool | None = False,
structured_output: bool = False,
reasoning_origin: str | None = None,
replays_openai_history: bool = False,
) -> BaseModelResponseIterator:
return OpenAiResponsesToChatCompletionStreamIterator(
@ -1628,6 +1719,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
sync_stream,
json_mode,
structured_output=structured_output,
reasoning_origin=reasoning_origin,
replays_openai_history=replays_openai_history,
)
@ -2021,9 +2113,11 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
sync_stream: bool,
json_mode: bool | None = False,
structured_output: bool = False,
reasoning_origin: str | None = None,
replays_openai_history: bool = False,
):
super().__init__(streaming_response, sync_stream, json_mode)
self._reasoning_origin: Final = reasoning_origin
self._replays_openai_history: Final = replays_openai_history
self._chat_completion_id: str | None = None
self._served_service_tier: str | None = None
@ -2071,6 +2165,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
def translate_responses_chunk_to_openai_stream(
parsed_chunk: dict | BaseModel,
tool_call_index_map: dict[int, int] | None = None, # mutable-ok: per-stream state, remapped in place
reasoning_origin: str | None = None,
replays_openai_history: bool = False,
structured_output: bool = False,
reasoning_summary_streamed: bool = False,
@ -2081,6 +2176,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
Args:
parsed_chunk: Dict containing the Responses API event chunk
tool_call_index_map: Per-stream output_index -> sequential tool_call index map
reasoning_origin: Origin tag stamped on streamed encrypted reasoning, so replays reach only that origin
replays_openai_history: Record the completed output's item sequence for the next turn
structured_output: Only the first message item became content
reasoning_summary_streamed: Whether earlier chunks already streamed reasoning summary text
@ -2294,7 +2390,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
)
terminal_reasoning_items_typed: Final = _as_chat_reasoning_items(
_reasoning_items_from_output_items(output_items)
_reasoning_items_from_output_items(_with_reasoning_origin(output_items, reasoning_origin))
)
output_payload: Final = cast(object, output_items) # cast-ok: untyped
tool_search_fields: Final = _tool_search_provider_fields(
@ -2371,6 +2467,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
else OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(
chunk,
tool_call_index_map=self._tool_call_index_map,
reasoning_origin=self._reasoning_origin,
replays_openai_history=self._replays_openai_history,
structured_output=self._structured_output,
reasoning_summary_streamed=self._reasoning_summary_streamed,

194
litellm/llms/bedrock/common_utils.py
View file

@ -12,7 +12,7 @@ import re
import warnings
from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from types import MappingProxyType, NoneType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias
from typing_extensions import (
@ -32,6 +32,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError
import litellm
from litellm import verbose_logger
from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES, OPENAI_CHAT_COMPLETION_PARAMS
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.core_helpers import lax_bool
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
@ -49,6 +50,8 @@ from litellm.llms.bedrock.request_metadata import bedrock_request_metadata_is_ow
from litellm.secret_managers.main import get_secret, get_secret_str
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.llms.bedrock import AWS_AUTH_PARAM_KEYS, AwsAuthParams
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
from litellm.types.utils import is_litellm_owned_kwarg
if TYPE_CHECKING:
from litellm.llms.custom_httpx.http_handler import HTTPHandler
@ -80,6 +83,7 @@ BedrockRoute = Literal[
"openai",
"mantle",
"chat_completions",
"responses",
]
@ -1369,7 +1373,8 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje
written onto the Converse body,
function tools (``tools`` or legacy ``functions``) on a model without
``supports_bedrock_runtime_chat_completions_tools_with_reasoning`` are rejected there unless
``reasoning_effort`` is exactly ``"none"``, and a ``response_format`` goes native only as
``reasoning_effort`` is exactly ``"none"`` (function-only ``tools`` on a model listing ``/v1/responses`` are
bridged to bedrock-runtime's native Responses API instead), and a ``response_format`` goes native only as
``{"type": "json_schema", "json_schema": ...}`` (a pydantic model is converted to that) on a model with
``supports_bedrock_runtime_chat_completions_response_format``: a schema on any other model is only
honored by Converse, and every ``json_object`` form (``response_schema`` included) keeps Converse's
@ -1387,6 +1392,12 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje
return True
if _response_format_needs_converse(model, request_params.get("response_format")):
return True
return _chat_completions_rejects_function_tools(model, request_params) and not _bridgeable_to_responses(
model, request_params
)
def _chat_completions_rejects_function_tools(model: str, request_params: Mapping[str, object]) -> bool:
if not (request_params.get("tools") or request_params.get("functions")):
return False
return (
@ -1395,12 +1406,166 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje
)
class _NamedFunction(TypedDict):
name: ReadOnly[str]
class _FunctionTool(TypedDict):
type: ReadOnly[Literal["function"]]
function: ReadOnly[_NamedFunction]
_FUNCTION_TOOLS_ADAPTER: Final = TypeAdapter(tuple[_FunctionTool, ...])
_OPENAI_FUNCTION_NAME: Final = re.compile(r"[a-zA-Z0-9_-]{1,64}")
def _function_tools_only(tools: object) -> bool:
if not isinstance(tools, (list, tuple)):
return False
try:
validated: Final = _FUNCTION_TOOLS_ADAPTER.validate_python(tools)
except ValidationError:
return False
return all(_OPENAI_FUNCTION_NAME.fullmatch(tool["function"]["name"]) for tool in validated)
_EXTRA_BODY_ADAPTER: Final = TypeAdapter(dict[str, object])
_RESPONSES_INPUTS_CHAT_HOOKS_NEVER_CHECK: Final = frozenset({"previous_response_id", "instructions", "prompt"})
_EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES: Final = frozenset(
{*ResponsesAPIOptionalRequestParams.__annotations__, "reasoning_effort", "reasoningSummary", "reasoning_summary"}
- {"tools", *_RESPONSES_INPUTS_CHAT_HOOKS_NEVER_CHECK}
)
_CHAT_PARAMS_THE_RESPONSES_BRIDGE_DROPS: Final = MappingProxyType(
{
"n": 1,
"presence_penalty": 0,
"frequency_penalty": 0,
"logprobs": False,
"top_logprobs": None,
"logit_bias": None,
"seed": None,
"modalities": None,
"audio": None,
"prediction": None,
"web_search_options": None,
"function_call": None,
"include_server_side_tool_invocations": None,
}
)
_ENCRYPTED_REASONING: Final = "reasoning.encrypted_content"
_INCLUDE_ADAPTER: Final = TypeAdapter(tuple[object, ...])
def _extra_body_bridgeable(extra_body: object) -> bool:
if extra_body is None:
return True
if not isinstance(extra_body, dict):
return False
validated: Final = _EXTRA_BODY_ADAPTER.validate_python(extra_body)
return (
validated.keys() <= _EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES
and isinstance(validated.get("store", False), bool)
and isinstance(validated.get("include"), (list, tuple, NoneType))
)
def _sets_a_chat_param_the_responses_bridge_drops(request_params: Mapping[str, object]) -> bool:
return any(
request_params.get(param) not in (None, no_op_value)
for param, no_op_value in _CHAT_PARAMS_THE_RESPONSES_BRIDGE_DROPS.items()
)
_CHAT_PARAM_NAMES: Final = frozenset(
{*DEFAULT_CHAT_COMPLETION_PARAM_VALUES, *OPENAI_CHAT_COMPLETION_PARAMS, "extra_body"}
)
def _provider_specific_param(name: str) -> bool:
return not (
name in _CHAT_PARAM_NAMES
or name in _EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES
or name.startswith("aws_")
or is_litellm_owned_kwarg(name)
)
def _passes_a_provider_specific_param(request_params: Mapping[str, object]) -> bool:
return any(value is not None and _provider_specific_param(name) for name, value in request_params.items())
def _bridgeable_to_responses(model: str, request_params: Mapping[str, object]) -> bool:
return (
not request_params.get("functions")
and request_params.get("aws_bedrock_client") is None
and _function_tools_only(request_params.get("tools"))
and _extra_body_bridgeable(request_params.get("extra_body"))
and not _passes_a_provider_specific_param(request_params)
and not _sets_a_chat_param_the_responses_bridge_drops(request_params)
and isinstance(request_params.get("reasoning_effort"), (str, NoneType))
and isinstance(request_params.get("store"), (bool, NoneType))
and isinstance(request_params.get("include"), (list, tuple, NoneType))
and _bedrock_serves_responses(model)
)
def _bedrock_serves_responses(model: str) -> bool:
region, model_id = split_bedrock_region_path(model)
return region is None and any(
_price_map_entry_lists_endpoint(_model_cost_entry(key), "/v1/responses")
for key in (model_id, f"bedrock/{model_id}")
)
def bedrock_responses_bridge_optional_params(optional_params: Mapping[str, object]) -> Mapping[str, object]:
if optional_params.get("store") is not None:
return optional_params
extra_body: Final = optional_params.get("extra_body")
extra_body_params: Final = _EXTRA_BODY_ADAPTER.validate_python(extra_body) if isinstance(extra_body, dict) else {}
include: Final = extra_body_params.get("include", optional_params.get("include"))
requested: Final = _INCLUDE_ADAPTER.validate_python(include or ())
return {
**optional_params,
**(
{"extra_body": {key: value for key, value in extra_body_params.items() if key != "include"}}
if "include" in extra_body_params
else {}
),
"store": False,
"include": [*requested, *(() if _ENCRYPTED_REASONING in requested else (_ENCRYPTED_REASONING,))],
}
def bedrock_responses_bridge_api_base(api_base: str | None, aws_bedrock_runtime_endpoint: object) -> str | None:
return (
(aws_bedrock_runtime_endpoint if isinstance(aws_bedrock_runtime_endpoint, str) else None)
or get_secret_str("AWS_BEDROCK_RUNTIME_ENDPOINT")
or api_base
)
def bedrock_responses_bridge_call_params(
optional_params: Mapping[str, object], litellm_params: Mapping[str, object], api_base: str | None
) -> tuple[Mapping[str, object], Mapping[str, object]]:
return (
bedrock_responses_bridge_optional_params(optional_params),
{
**litellm_params,
"api_base": bedrock_responses_bridge_api_base(api_base, litellm_params.get("aws_bedrock_runtime_endpoint")),
},
)
def _chat_completions_unless_converse_needed(
model: str, request_params: Mapping[str, object] | None
) -> Literal["converse", "chat_completions"]:
if request_params is not None and bedrock_request_needs_converse(model, request_params):
) -> Literal["converse", "chat_completions", "responses"]:
if request_params is None:
return "chat_completions"
if bedrock_request_needs_converse(model, request_params):
return "converse"
return "chat_completions"
if not _chat_completions_rejects_function_tools(model, request_params):
return "chat_completions"
return "converse" if request_params.get("_litellm_responses_api_bridge_allowed") is False else "responses"
def bedrock_route_for_request(
@ -1417,6 +1582,17 @@ def bedrock_route_for_request(
)
def bedrock_request_bridges_to_responses(
custom_llm_provider: str | None,
model: str,
request_params: Mapping[str, object],
additional_drop_params: Sequence[str] | None,
) -> bool:
if custom_llm_provider != "bedrock":
return False
return bedrock_route_for_request(model, request_params, additional_drop_params) == "responses"
def strip_bedrock_throughput_suffix(model: str) -> str:
"""Strip throughput tier suffixes and context window suffixes from Bedrock model names."""
import re
@ -1999,9 +2175,11 @@ class BedrockModelInfo(BaseLLMModelInfo):
Grok and GPT 5.6 and newer go to bedrock-runtime's native OpenAI Chat Completions by
default (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in;
``request_params`` (the caller's chat params) sends such a request to Converse when it needs a
feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family
model stays on Converse without the prefix. A model the cost map does not list as Converse goes to
Invoke only when its id names an Invoke provider, and to Converse otherwise.
feature only Converse serves, to ``responses`` (the chat-to-Responses bridge onto bedrock-runtime's
native ``/openai/v1/responses``) when it carries function tools with reasoning on, and ``converse/``
pins a model to Converse. Every other OpenAI-family model stays on Converse without the prefix. A model
the cost map does not list as Converse goes to Invoke only when its id names an Invoke provider, and to
Converse otherwise.
"""
named_route: Final = BedrockModelInfo.named_bedrock_route(model, request_params)
if named_route is not None:

136
litellm/main.py
View file

@ -57,6 +57,7 @@ from litellm.utils import (
peek_reasoning_summary_aliases,
reasoning_effort_with_summary_alias,
strip_reasoning_summary_aliases_from_optional_params,
with_reasoning_summary_in_effort,
)
if TYPE_CHECKING:
@ -123,6 +124,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
)
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
from litellm.litellm_core_utils.reasoning_effort_capability import (
RequestedReasoningEffort,
model_group_from_metadata,
reasoning_effort_needs_allowing,
resolve_chat_reasoning,
@ -140,6 +142,8 @@ from litellm.llms.base_llm.chat.transformation import with_attribution_headers
from litellm.llms.bedrock.common_utils import (
BedrockModelInfo,
NativeOutputFallback,
bedrock_request_bridges_to_responses,
bedrock_responses_bridge_call_params,
bedrock_route_for_request,
bedrock_strict_tools_fallback,
deployment_model_info,
@ -5475,6 +5479,54 @@ def _complete_databricks_agent(ctx: CompletionDispatchContext) -> _CompletionDis
)
def _responses_bridge_request_params(
*,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
bridges_bedrock: bool,
resolved_reasoning_effort: RequestedReasoningEffort,
model: str,
base_model: object,
custom_llm_provider: str,
api_base: str | None,
deployment_model_info: Mapping[str, object] | None,
drop_params: bool,
additional_drop_params: object,
) -> tuple[Mapping[str, object], Mapping[str, object]]:
from litellm.responses.utils import (
RefusedReasoningFields,
bridge_reasoning,
raise_refused_reasoning_fields,
)
bridge_optional_params, bridge_litellm_params = (
bedrock_responses_bridge_call_params(optional_params, litellm_params, api_base)
if bridges_bedrock
else (optional_params, litellm_params)
)
stripped_optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(bridge_optional_params)
if not isinstance(resolved_reasoning_effort, dict):
return (
with_reasoning_summary_in_effort(stripped_optional_params, resolved_reasoning_effort, rs_val),
bridge_litellm_params,
)
bridged_reasoning: Final = bridge_reasoning(
requested=resolved_reasoning_effort,
effort_kept="reasoning_effort" in stripped_optional_params,
summary_alias=rs_val,
model=model,
base_model=base_model if isinstance(base_model, str) else None,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
deployment_model_info=deployment_model_info,
drop_params=drop_params,
additional_drop_params=additional_drop_params,
)
if isinstance(bridged_reasoning, RefusedReasoningFields):
raise_refused_reasoning_fields(bridged_reasoning, model, custom_llm_provider)
return {**stripped_optional_params, "reasoning_effort": bridged_reasoning}, bridge_litellm_params
@tracer.wrap()
@client
def completion(
@ -6028,6 +6080,7 @@ def completion(
"allowed_openai_params": allowed_openai_params,
"base_model": base_model,
}
bedrock_responses_bridge_allowed: Final = acompletion is True and not skip_responses_api_bridge
provider_reasoning_effort: Final = reasoning_effort_with_summary_alias(
resolved_reasoning_effort,
provider_params,
@ -6039,7 +6092,12 @@ def completion(
mapping_special_params: Final = MappingProxyType(dict(provider_params))
def optional_params_for_model_info(deployment_info: Mapping[str, object] | None) -> dict[str, object]:
return get_optional_params(**mapping_args, model_info=deployment_info, **mapping_special_params)
return get_optional_params(
**mapping_args,
model_info=deployment_info,
**mapping_special_params,
_litellm_responses_api_bridge_allowed=bedrock_responses_bridge_allowed,
)
validated_model_info: Final = (
_MODEL_INFO_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else None
@ -6271,48 +6329,44 @@ def completion(
deployment_model_info=typed_deployment_model_info,
)
request_params: Final = MappingProxyType(
_OPTIONAL_PARAMS_ADAPTER.validate_python(
{
**optional_param_args,
**provider_params,
"_litellm_responses_api_bridge_allowed": bedrock_responses_bridge_allowed,
}
)
)
bedrock_bridges_to_responses: Final = custom_llm_provider == "bedrock" and bedrock_request_bridges_to_responses(
custom_llm_provider,
model,
request_params,
_ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(kwargs.get("additional_drop_params") or ()),
)
# Use base_model (the true underlying model) for Azure model-type
# detection when the deployment name differs from the model name.
_azure_detection_model: Final = base_model or model
if responses_api_model_info.get("mode") == "responses" and not skip_responses_api_bridge:
if (
responses_api_model_info.get("mode") == "responses" or bedrock_bridges_to_responses
) and not skip_responses_api_bridge:
from litellm.completion_extras import responses_api_bridge
optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params)
if isinstance(resolved_reasoning_effort, dict):
from litellm.responses.utils import (
RefusedReasoningFields,
bridge_reasoning,
raise_refused_reasoning_fields,
)
bridged_reasoning: Final = bridge_reasoning(
requested=resolved_reasoning_effort,
effort_kept="reasoning_effort" in optional_params,
summary_alias=rs_val,
model=model,
base_model=base_model if isinstance(base_model, str) else None,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
deployment_model_info=typed_deployment_model_info,
drop_params=drop_unsupported_reasoning,
additional_drop_params=routing_kwargs.get("additional_drop_params"),
)
if isinstance(bridged_reasoning, RefusedReasoningFields):
raise_refused_reasoning_fields(bridged_reasoning, model, custom_llm_provider)
optional_params["reasoning_effort"] = bridged_reasoning
elif rs_val is not None:
eff: Final = optional_params.get("reasoning_effort", resolved_reasoning_effort)
if isinstance(eff, dict):
optional_params["reasoning_effort"] = {**eff, "summary": rs_val}
elif eff is not None:
optional_params["reasoning_effort"] = {
"effort": eff,
"summary": rs_val,
}
else:
optional_params["reasoning_effort"] = {"summary": rs_val}
bridged_optional_params, bridge_litellm_params = _responses_bridge_request_params(
optional_params=optional_params,
litellm_params=litellm_params,
bridges_bedrock=bedrock_bridges_to_responses,
resolved_reasoning_effort=resolved_reasoning_effort,
model=model,
base_model=base_model,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
deployment_model_info=typed_deployment_model_info,
drop_params=drop_unsupported_reasoning,
additional_drop_params=routing_kwargs.get("additional_drop_params"),
)
return responses_api_bridge.completion( # pyright: ignore[reportReturnType] # bridge returns a coroutine on the acompletion path; awaited by the async caller
model=model,
@ -6323,8 +6377,8 @@ def completion(
api_base=api_base,
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
optional_params=bridged_optional_params,
litellm_params=bridge_litellm_params,
timeout=timeout,
client=client, # pass AsyncOpenAI, OpenAI client
custom_llm_provider=custom_llm_provider,
@ -6364,9 +6418,7 @@ def completion(
optional_params_for_model_info=optional_params_for_model_info,
organization=organization,
provider_config=provider_config,
request_params=MappingProxyType(
_OPTIONAL_PARAMS_ADAPTER.validate_python({**optional_param_args, **provider_params})
),
request_params=request_params,
shared_session=shared_session,
stream=stream,
temperature=temperature,

27
litellm/utils.py
View file

@ -5547,6 +5547,7 @@ def get_optional_params(
prompt_cache_key: str | None = None,
base_model: str | None = None,
model_info: Mapping[str, object] | None = None,
_litellm_responses_api_bridge_allowed: bool = True,
**kwargs: object,
):
drop_params = normalize_drop_params(drop_params) # rebind-ok: config and DB deployments pass "true" as a string
@ -5558,6 +5559,7 @@ def get_optional_params(
# OpenAI param.
passed_params.pop("base_model", None)
passed_params.pop("model_info", None)
passed_params.pop("_litellm_responses_api_bridge_allowed")
provider_config: BaseConfig | None = None
if custom_llm_provider is not None and custom_llm_provider in [provider.value for provider in LlmProviders]:
provider_config = ProviderConfigManager.get_provider_chat_config(
@ -5624,7 +5626,11 @@ def get_optional_params(
)
bedrock_route: Final = (
_bedrock_route_for_request(model, passed_params, additional_drop_params)
_bedrock_route_for_request(
model,
{**passed_params, "_litellm_responses_api_bridge_allowed": _litellm_responses_api_bridge_allowed},
additional_drop_params,
)
if custom_llm_provider == "bedrock"
else None
)
@ -11826,8 +11832,8 @@ def reasoning_effort_with_summary_alias(
def strip_reasoning_summary_aliases_from_optional_params(
optional_params: dict,
) -> tuple[dict, object | None]:
optional_params: Mapping[str, object],
) -> tuple[dict[str, object], object | None]:
"""Copy optional_params; remove reasoningSummary aliases from top-level and extra_body."""
op: Final = dict(optional_params)
rs_val = op.pop("reasoningSummary", None)
@ -11850,6 +11856,21 @@ def strip_reasoning_summary_aliases_from_optional_params(
return op, rs_val
def with_reasoning_summary_in_effort(
optional_params: Mapping[str, object], resolved_reasoning_effort: object, reasoning_summary: object | None
) -> Mapping[str, object]:
if isinstance(resolved_reasoning_effort, dict) and "summary" in resolved_reasoning_effort:
return {**optional_params, "reasoning_effort": resolved_reasoning_effort}
if reasoning_summary is None:
return optional_params
effort: Final = optional_params.get("reasoning_effort", resolved_reasoning_effort)
if isinstance(effort, dict):
return {**optional_params, "reasoning_effort": {**effort, "summary": reasoning_summary}}
if effort is not None:
return {**optional_params, "reasoning_effort": {"effort": effort, "summary": reasoning_summary}}
return {**optional_params, "reasoning_effort": {"summary": reasoning_summary}}
def get_non_default_transcription_params(kwargs: Mapping[str, object]) -> dict[str, object]:
from litellm.constants import OPENAI_TRANSCRIPTION_PARAMS

90
tests/integration/messages_endpoint/providers/bedrock/test_bedrock_messages_gpt_chat_completions_wire.py
View file

@ -4,7 +4,15 @@ from collections.abc import Mapping
from typing import Final
import anthropic
from integration._support.bedrock_runtime_peer import NATIVE_CHAT, answer, body_of, marker_of, respond, target_of
from integration._support.bedrock_runtime_peer import (
NATIVE_CHAT,
NATIVE_RESPONSES,
answer,
body_of,
marker_of,
respond,
target_of,
)
from integration._support.client import Gateway, Scenario, eventually
from integration._support.database import read_rows
from integration._support.wire import Request, Wire, wire_server
@ -14,6 +22,7 @@ BEDROCK_MODEL: Final = "us.openai.gpt-5.6-sol"
TOKEN: Final = "synthetic-bedrock-bearer"
NO_CACHE: Final[Mapping[str, JsonValue]] = {"cache": {"no-cache": True}}
ANTHROPIC_VERSION: Final[Mapping[str, str]] = {"anthropic-version": "2023-06-01"}
TOOL_SCHEMA: Final[dict[str, object]] = {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}
def _question(marker: str) -> str:
@ -40,6 +49,41 @@ def _native_body(wire: Wire, marker: str) -> Mapping[str, JsonValue]:
return body_of(received[0])
def _bridged_body(wire: Wire, marker: str) -> Mapping[str, JsonValue]:
received: Final = _carrying(wire, marker)
assert [(request.method, target_of(request)) for request in received] == [("POST", NATIVE_RESPONSES)]
assert received[0].headers["authorization"] == f"Bearer {TOKEN}", received[0].headers
body: Final = body_of(received[0])
assert (body["model"], body["reasoning"], body["max_output_tokens"]) == (
BEDROCK_MODEL,
{"effort": "medium"},
4096,
), body
assert (body["store"], body["include"]) == (False, ["reasoning.encrypted_content"]), body
assert [(tool["type"], tool["name"], tool["parameters"]) for tool in _tools_of(body)] == [
("function", "lookup_invoice", TOOL_SCHEMA)
], body
return body
def _tools_of(body: Mapping[str, JsonValue]) -> list[dict[str, JsonValue]]:
tools: Final = body["tools"]
assert isinstance(tools, list), body
return [tool for tool in tools if isinstance(tool, dict)]
def _bridged_spend_row(model: str) -> dict[str, JsonValue]:
rows: Final = eventually(
lambda: read_rows(
'SELECT status, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE model_group=%s',
(model,),
),
lambda found: len(found) == 1,
seconds=70,
)
return rows[0]
def _native_request(marker: str, max_tokens: int, effort: str) -> Mapping[str, JsonValue]:
return {
"model": BEDROCK_MODEL,
@ -122,6 +166,50 @@ def test_anthropic_sdk_stream_with_thinking_budget_is_served_by_native_chat_comp
assert _spend_rows(identity, 1) == [_success_row(identity, model, cache_hit="False")]
def test_anthropic_sdk_tools_with_thinking_are_bridged_to_native_responses(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
client: Final = anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0)
message: Final = client.messages.create(
model=model,
max_tokens=4096,
thinking={"type": "enabled", "budget_tokens": 2048},
tools=[{"name": "lookup_invoice", "description": "Look up an invoice", "input_schema": TOOL_SCHEMA}],
messages=[{"role": "user", "content": _question(marker)}],
extra_body=NO_CACHE,
)
assert "stream" not in _bridged_body(wire, marker)
assert [(block.type, getattr(block, "text", None)) for block in message.content] == [("text", answer(marker))]
assert (message.usage.input_tokens, message.usage.output_tokens) == (30, 5), message
assert _bridged_spend_row(model) == {"status": "success", "prompt_tokens": 30, "completion_tokens": 5}
def test_anthropic_sdk_stream_with_tools_and_thinking_is_bridged_to_native_responses(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
client: Final = anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0)
stream: Final = client.messages.create(
model=model,
max_tokens=4096,
thinking={"type": "enabled", "budget_tokens": 2048},
tools=[{"name": "lookup_invoice", "description": "Look up an invoice", "input_schema": TOOL_SCHEMA}],
messages=[{"role": "user", "content": _question(marker)}],
extra_body=NO_CACHE,
stream=True,
)
events: Final = list(stream)
assert _bridged_body(wire, marker)["stream"] is True
assert events[0].type == "message_start" and events[-1].type == "message_stop", events
assert "".join(
event.delta.text
for event in events
if event.type == "content_block_delta" and event.delta.type == "text_delta"
) == answer(marker)
assert _bridged_spend_row(model) == {"status": "success", "prompt_tokens": 30, "completion_tokens": 5}
def test_raw_thinking_summary_reaches_native_chat_completions_as_the_plain_effort(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:

269
tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py
View file

@ -8,18 +8,21 @@ from urllib.parse import quote
import httpx
import openai
import pytest
from integration._support.bedrock_runtime_peer import answer, respond, target_of
from integration._support.client import Gateway, Scenario, eventually
from integration._support import responses_vendor as rv
from integration._support.bedrock_runtime_peer import NATIVE_RESPONSES, answer, respond, target_of
from integration._support.client import Gateway, Scenario, eventually, list_value, object_value
from integration._support.database import read_rows
from integration._support.sigv4 import signature
from integration._support.wire import Request, Wire, wire_server
from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
from openai.types.chat import ChatCompletionChunk, ChatCompletionFunctionToolParam, ChatCompletionMessageParam
from openai.types.chat.chat_completion_chunk import ChoiceDelta
from pydantic import JsonValue, TypeAdapter
GPT: Final = "us.openai.gpt-5.6-sol"
GLOBAL_GPT: Final = "global.openai.gpt-5.6-sol"
GPT_OSS: Final = "openai.gpt-oss-120b-1:0"
OTHER_GPT: Final = "global.openai.gpt-6.1-sol"
OPENAI_CODEX: Final = "gpt-5.3-codex"
TOKEN: Final = "synthetic-bedrock-bearer"
ACCESS_KEY: Final = "AKIASYNTHETICKEY0001"
SECRET_KEY: Final = "synthetic-secret-key-for-testing"
@ -45,6 +48,15 @@ TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType(
},
}
)
SDK_TOOL: Final[ChatCompletionFunctionToolParam] = {
"type": "function",
"function": {
"name": "lookup_invoice",
"description": "Look up an invoice",
"parameters": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]},
},
}
ENCRYPTED_REASONING: Final = "reasoning.encrypted_content"
CONVERSE_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType(
{
"toolSpec": {
@ -159,10 +171,27 @@ def _spend_row(identity: str) -> dict[str, JsonValue]:
return rows[0]
def _only_spend_row_of(model: str) -> dict[str, JsonValue]:
rows: Final = eventually(
lambda: read_rows(
'SELECT model_group, status, prompt_tokens, completion_tokens, api_base FROM "LiteLLM_SpendLogs"'
" WHERE model_group=%s",
(model,),
),
lambda found: len(found) == 1,
seconds=70,
)
return rows[0]
def _success_row(model: str, api_base: str) -> dict[str, JsonValue]:
return {"model_group": model, "status": "success", "prompt_tokens": 9, "completion_tokens": 5, "api_base": api_base}
def _bridged_success_row(model: str, wire_url: str) -> dict[str, JsonValue]:
return {**_success_row(model, f"{wire_url}{NATIVE_RESPONSES}"), "prompt_tokens": 30}
def _delta_text(delta: ChoiceDelta, field: str) -> str:
value: Final = delta.model_dump().get(field)
return value if isinstance(value, str) else ""
@ -387,17 +416,247 @@ def test_json_schema_response_format_is_forwarded_natively(gateway: Gateway) ->
assert _spend_row(f"chatcmpl-{marker}") == _success_row(model, f"{wire.url}{NATIVE_TARGET}")
def test_tools_while_reasoning_keep_converse(gateway: Gateway) -> None:
def test_tools_while_reasoning_are_bridged_to_native_responses(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high")
payload: Final = _payload(response)
assert payload["id"] == f"resp_upstream_{marker}", response.text
choices: Final = payload["choices"]
assert isinstance(choices, list) and len(choices) == 1, response.text
assert choices[0]["finish_reason"] == "stop", response.text
assert choices[0]["message"]["content"] == answer(marker), response.text
body: Final = _body(_converse_request(wire, NATIVE_RESPONSES))
assert body["model"] == GPT, body
assert body["reasoning"] == {"effort": "high"}, body
assert (body["store"], body["include"]) == (False, [ENCRYPTED_REASONING]), body
assert [(tool["type"], tool["name"], tool["parameters"]) for tool in body["tools"]] == [
("function", "lookup_invoice", dict(TOOL_PARAMETERS))
], body
assert _upstream_requests_mentioning(gateway, marker) == [], response.text
assert _only_spend_row_of(model) == _bridged_success_row(model, wire.url)
def test_streamed_tools_while_reasoning_are_bridged_to_native_responses(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
stream: Final = _openai_client(gateway).chat.completions.create(
model=model,
messages=_sdk_messages(marker),
tools=[SDK_TOOL],
reasoning_effort="high",
stream=True,
stream_options={"include_usage": True},
extra_body={"cache": {"no-cache": True}},
)
chunks: Final = list(stream)
assert _joined(chunks, "content") == answer(marker), chunks
usage: Final = chunks[-1].usage
assert usage is not None and (usage.prompt_tokens, usage.completion_tokens) == (30, 5), chunks[-1]
body: Final = _body(_converse_request(wire, NATIVE_RESPONSES))
assert (body["model"], body["stream"], body["reasoning"]) == (GPT, True, {"effort": "high"}), body
assert (body["store"], body["include"]) == (False, [ENCRYPTED_REASONING]), body
assert [tool["name"] for tool in body["tools"]] == ["lookup_invoice"], body
assert _only_spend_row_of(model) == _bridged_success_row(model, wire.url)
def test_tools_while_reasoning_are_bridged_with_the_deployments_rounded_effort(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model_info={"reasoning_effort_levels": ["low", "high"]},
**{**GPT_DEPLOYMENT, "aws_bedrock_runtime_endpoint": wire.url},
)
response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="medium")
assert _payload(response)["id"] == f"resp_upstream_{marker}", response.text
body: Final = _body(_converse_request(wire, NATIVE_RESPONSES))
assert (body["model"], body["reasoning"]) == (GPT, {"effort": "high"}), body
def _reasoning_deployment(scenario: Scenario, wire: Wire, target: str) -> str:
if target == OPENAI_CODEX:
return scenario.model(model=f"openai/{OPENAI_CODEX}", api_base=wire.url, api_key="synthetic-openai-key")
return _deployment(scenario, wire, model=f"bedrock/{target}")
def _message(payload: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
return object_value(object_value(list_value(payload["choices"])[0])["message"])
def _reasoning_turn(gateway: Gateway, model: str, marker: str) -> list[dict[str, JsonValue]]:
response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high")
return rv.ITEMS.validate_python(_message(_payload(response))["reasoning_items"])
def _replay(
gateway: Gateway, model: str, marker: str, reasoning_items: Sequence[Mapping[str, JsonValue]]
) -> dict[str, JsonValue]:
history: Final[list[JsonValue]] = [
{"role": "user", "content": "Pick an invoice."},
{"role": "assistant", "content": "Invoice 7", "reasoning_items": [dict(item) for item in reasoning_items]},
{"role": "user", "content": _prompt(marker)},
]
return _payload(
gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": history,
"tools": [dict(TOOL)],
"reasoning_effort": "high",
"cache": {"no-cache": True},
},
)
)
def _vendor_reasoning_item(item_id: str) -> dict[str, JsonValue]:
return {
"type": "reasoning",
"id": item_id,
"summary": [],
"encrypted_content": f"gAAAAA-vendor-{item_id.removeprefix('rs_')}",
}
@pytest.mark.parametrize("replay_to", [GLOBAL_GPT, GPT], ids=["to-another-inference-profile", "to-the-same-profile"])
def test_bridged_reasoning_replays_unchanged_to_the_bedrock_model_that_minted_it(
gateway: Gateway, replay_to: str
) -> None:
marker: Final = uuid.uuid4().hex
follow_up: Final = uuid.uuid4().hex
vendor: Final = rv.ResponsesVendor()
with wire_server(vendor.respond) as wire, gateway.scenario() as scenario:
minter: Final = _deployment(scenario, wire)
replayer: Final = minter if replay_to == GPT else _reasoning_deployment(scenario, wire, replay_to)
(item,) = _reasoning_turn(gateway, minter, marker)
(minted,) = vendor.minted
assert item["id"] == minted, item
assert target_of(_only_request(wire)) == NATIVE_RESPONSES
message: Final = _message(_replay(gateway, replayer, follow_up, [item]))
assert message["content"] == rv.answer(follow_up), message
body: Final = _body(_converse_request(wire, NATIVE_RESPONSES))
assert body["model"] == replay_to, body
assert rv.reasoning_items(body) == [_vendor_reasoning_item(minted)], body["input"]
@pytest.mark.parametrize(
("minted_on", "replayed_to"),
[(GPT, OTHER_GPT), (GPT, OPENAI_CODEX), (OPENAI_CODEX, GPT)],
ids=["bedrock-to-another-bedrock-model", "bedrock-to-openai", "openai-to-bedrock"],
)
def test_reasoning_is_not_replayed_to_a_model_that_did_not_mint_it(
gateway: Gateway, minted_on: str, replayed_to: str
) -> None:
marker: Final = uuid.uuid4().hex
follow_up: Final = uuid.uuid4().hex
vendor: Final = rv.ResponsesVendor()
with wire_server(vendor.respond) as wire, gateway.scenario() as scenario:
minter: Final = _reasoning_deployment(scenario, wire, minted_on)
replayer: Final = _reasoning_deployment(scenario, wire, replayed_to)
(item,) = _reasoning_turn(gateway, minter, marker)
assert item["id"] == vendor.minted[0], item
wire.drain()
message: Final = _message(_replay(gateway, replayer, follow_up, [item]))
assert message["content"] == rv.answer(follow_up), message
body: Final = _body(_only_request(wire))
assert body["model"] == replayed_to, body
assert rv.reasoning_items(body) == [], body["input"]
assert [entry.get("role") for entry in rv.ITEMS.validate_python(body["input"])] == [
"user",
"assistant",
"user",
], body["input"]
def test_tools_while_reasoning_with_a_provider_specific_field_keep_converse_and_reach_the_model(
gateway: Gateway,
) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high", foo_bar=1)
assert _message(_payload(response))["content"] == answer(marker), response.text
body: Final = _body(_converse_request(wire))
assert object_value(body["additionalModelRequestFields"])["foo_bar"] == 1, body
assert body["toolConfig"] == {"tools": [dict(CONVERSE_TOOL)]}, body
def test_tools_while_reasoning_with_a_non_boolean_store_are_refused_like_converse(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high", store="false")
assert response.status_code == 400, response.text
assert "store" in response.text, response.text
assert wire.drain() == (), response.text
def test_tools_while_reasoning_on_a_sigv4_deployment_are_bridged_signed_with_its_keys(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"bedrock/{GPT}",
api_key=None,
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
aws_region_name="us-east-1",
aws_bedrock_runtime_endpoint=wire.url,
)
response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high")
assert _payload(response)["id"] == f"resp_upstream_{marker}", response.text
request: Final = _only_request(wire)
assert (request.method, target_of(request)) == ("POST", NATIVE_RESPONSES), request.target
_assert_sigv4_signed(request, NATIVE_RESPONSES)
assert _body(request)["reasoning"] == {"effort": "high"}, request.body
assert _only_spend_row_of(model) == _bridged_success_row(model, wire.url)
@pytest.mark.parametrize("suffix", ["", "/openai/v1"])
def test_tools_while_reasoning_are_bridged_to_the_api_base_without_a_runtime_endpoint(
gateway: Gateway, suffix: str
) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model_info=None, **{**GPT_DEPLOYMENT, "api_base": f"{wire.url}{suffix}"})
response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high")
request: Final = _only_request(wire)
assert (request.method, request.target) == ("POST", NATIVE_RESPONSES), response.text
assert _payload(response)["id"] == f"resp_upstream_{marker}", response.text
assert _body(request)["reasoning"] == {"effort": "high"}, request.body
assert _only_spend_row_of(model) == _bridged_success_row(model, wire.url)
def test_tools_while_reasoning_effort_is_an_object_keep_converse(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(
gateway, model, marker, tools=[dict(TOOL)], reasoning_effort={"effort": "high", "summary": "auto"}
)
payload: Final = _payload(response)
assert payload["choices"][0]["message"]["content"] == answer(marker), response.text
assert _body(_converse_request(wire))["toolConfig"] == {"tools": [dict(CONVERSE_TOOL)]}
assert _spend_row(str(payload["id"])) == _success_row(model, f"{wire.url}{CONVERSE_TARGET}")
def test_tools_while_reasoning_with_a_name_openai_rejects_keep_converse(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
tool: Final = {**TOOL, "function": {**TOOL["function"], "name": "lookup invoice!"}}
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, tools=[tool], reasoning_effort="high")
payload: Final = _payload(response)
assert payload["choices"] == [
{"finish_reason": "stop", "index": 0, "message": {"content": answer(marker), "role": "assistant"}}
], response.text
body: Final = _body(_converse_request(wire))
assert body["toolConfig"] == {"tools": [CONVERSE_TOOL]}, body
assert body["toolConfig"] == {
"tools": [{"toolSpec": {**CONVERSE_TOOL["toolSpec"], "name": "lookup_invoice_"}}]
}, body
assert body["additionalModelRequestFields"] == {"reasoning": {"effort": "high"}}, body
assert _spend_row(str(payload["id"])) == _success_row(model, f"{wire.url}{CONVERSE_TARGET}")

61
tests/integration/sdk/test_bedrock_tools_while_reasoning_sync_sdk.py Normal file
View file

@ -0,0 +1,61 @@
import uuid
from collections.abc import Mapping
from typing import Final
import pytest
from integration._support.bedrock_runtime_peer import answer, respond, target_of
from integration._support.client import eventually
from integration._support.wire import wire_server
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.utils import Choices, ModelResponse
_MODEL_ID: Final = "us.openai.gpt-5.6-sol"
_TOOL: Final = {
"type": "function",
"function": {
"name": "lookup_invoice",
"description": "Look up an invoice",
"parameters": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]},
},
}
class _SuccessRecorder(CustomLogger):
successes: tuple[tuple[object, object], ...] = ()
def log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self.successes = (*self.successes, (kwargs.get("call_type"), getattr(response_obj, "id", None)))
def test_sync_completion_with_tools_while_reasoning_keeps_converse_and_logs_one_success(
monkeypatch: pytest.MonkeyPatch,
) -> None:
recorder: Final = _SuccessRecorder()
monkeypatch.setattr(litellm, "callbacks", [recorder])
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire:
response: Final = litellm.completion(
model=f"bedrock/{_MODEL_ID}",
messages=[{"role": "user", "content": f"synthetic native request marker-{marker}"}],
tools=[_TOOL],
reasoning_effort="high",
api_key="synthetic-bedrock-bearer",
aws_region_name="us-east-1",
aws_bedrock_runtime_endpoint=wire.url,
num_retries=0,
)
sent: Final = [(request.method, target_of(request)) for request in wire.drain()]
assert sent == [("POST", f"/model/{_MODEL_ID}/converse")], sent
assert isinstance(response, ModelResponse), type(response)
choice: Final = response.choices[0]
assert isinstance(choice, Choices), choice
assert choice.message.content == answer(marker), choice
logged: Final = eventually(
lambda: recorder.successes,
lambda successes: any(logged_id == response.id for _, logged_id in successes),
)
assert logged == (("completion", response.id),), logged

6
tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_converse.py
View file

@ -11,7 +11,7 @@ GPT_6_1_SOL_TOOL_SEARCH_TEST_CASE: Final = TranslationTestCase(
scenario="tool_search",
litellm_endpoint="/v1/messages",
litellm_request={
"model": "bedrock/global.openai.gpt-6.1-sol",
"model": "bedrock/converse/global.openai.gpt-6.1-sol",
"max_tokens": 1024,
"thinking": {"type": "adaptive"},
"output_config": {"effort": "low"},
@ -54,7 +54,7 @@ GPT_6_1_SOL_TOOL_SEARCH_TEST_CASE: Final = TranslationTestCase(
"cache": {"no-cache": True},
},
expected_provider_endpoint="/model/global.openai.gpt-6.1-sol/converse",
expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"},
expected_provider_headers={"authorization": "Bearer synthetic-agent-key", "content-type": "application/json"},
expected_provider_request={
"additionalModelRequestFields": {"reasoning": {"effort": "low"}},
"inferenceConfig": {"maxTokens": 1024},
@ -139,7 +139,7 @@ GPT_6_1_SOL_TOOL_SEARCH_TEST_CASE: Final = TranslationTestCase(
"id": ANY,
"type": "message",
"role": "assistant",
"model": "bedrock/global.openai.gpt-6.1-sol",
"model": "bedrock/converse/global.openai.gpt-6.1-sol",
"stop_sequence": None,
"usage": {"input_tokens": 100, "output_tokens": 15},
"content": [{"type": "text", "text": "The web search tool is loaded and ready to use."}],

51
tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py
View file

@ -10,6 +10,9 @@ from litellm.completion_extras.litellm_responses_transformation.handler import (
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.responses.litellm_completion_transformation.streaming_iterator import (
LiteLLMCompletionStreamingIterator,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ModelResponse
@ -266,6 +269,54 @@ def test_completion_streams_completed_model_response():
)
def _chat_stream_behind_responses() -> LiteLLMCompletionStreamingIterator:
return LiteLLMCompletionStreamingIterator(
model="claude-haiku-5-5",
litellm_custom_stream_wrapper=litellm.completion(
model="anthropic/claude-haiku-5-5",
messages=[{"role": "user", "content": "hi"}],
mock_response="pong",
stream=True,
),
request_input="hi",
responses_api_request={},
custom_llm_provider="anthropic",
)
def test_completion_streams_a_chat_stream_bridged_back_from_responses():
bridge = ResponsesToCompletionBridgeHandler()
with (
patch.object(
bridge.transformation_handler,
"transform_request",
return_value={"model": "claude-haiku-5-5", "input": "hi"},
),
patch("litellm.responses", return_value=_chat_stream_behind_responses()),
):
result = bridge.completion(**_bridge_kwargs(stream=True))
assert "".join(chunk.choices[0].delta.content or "" for chunk in result) == "pong"
@pytest.mark.asyncio
async def test_acompletion_streams_a_chat_stream_bridged_back_from_responses():
bridge = ResponsesToCompletionBridgeHandler()
with (
patch.object(
bridge.transformation_handler,
"transform_request",
return_value={"model": "claude-haiku-5-5", "input": "hi"},
),
patch("litellm.aresponses", new=AsyncMock(return_value=_chat_stream_behind_responses())),
):
result = await bridge.acompletion(**_bridge_kwargs(stream=True))
assert "".join([chunk.choices[0].delta.content or "" async for chunk in result]) == "pong"
_PROVIDER_NATIVE_MODEL_CASES = [
("perplexity", "perplexity/kimi-k3", "perplexity/kimi-k3"),
("perplexity", "openai/gpt-5.2", "openai/gpt-5.2"),

31
tests/unit/completion_extras/test_responses_bridge_provider_propagation.py
View file

@ -13,6 +13,7 @@ from litellm.completion_extras.litellm_responses_transformation.handler import (
ResponsesToCompletionBridgeHandler,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.utils import ModelResponse
MODEL = "openai.gpt-5.5"
@ -193,3 +194,33 @@ async def test_acompletion_without_cache_field_is_served_from_cache(_bridged_cac
assert len(_bridged_cache_edge.calls) == 1
assert [first.id, second.id] == ["resp_1", "resp_1"]
@pytest.mark.parametrize(
"bridged_request",
[
{"model": "openai/gpt-5.5-pro"},
{"model": "azure/gpt-5-pro", "api_version": "2025-04-01-preview"},
{**_GPT_6_REQUEST, "reasoning_effort": "low"},
],
ids=["openai_responses_mode", "azure_responses_mode", "azure_tools_with_reasoning"],
)
async def test_async_bridged_call_is_sent_through_the_callers_client(bridged_request):
sent: Final[list[httpx.Request]] = []
async def handle(request: httpx.Request) -> httpx.Response:
sent.append(request)
return httpx.Response(200, json=_responses_api_body(1))
response = await litellm.acompletion(
**{
"messages": [{"role": "user", "content": "hi"}],
"api_key": "x",
**bridged_request,
"api_base": "http://127.0.0.1:9",
},
client=AsyncHTTPHandler(transport=httpx.MockTransport(handle)),
)
assert [(request.url.host, request.url.path.endswith("/responses")) for request in sent] == [("127.0.0.1", True)]
assert response.choices[0].message.content == "hi 1"

618
tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py
View file

@ -3,6 +3,8 @@
import json
import logging
import re
from collections.abc import Iterable, Mapping
from typing import Final
import httpx
import pytest
@ -25,7 +27,7 @@ from litellm.llms.bedrock.common_utils import (
bedrock_runtime_chat_completions_is_default,
get_bedrock_chat_config,
)
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
APPLICATION_INFERENCE_PROFILE_ARN = "arn:aws:bedrock:us-west-2:123412341234:application-inference-profile/a1b2c3"
@ -261,6 +263,16 @@ def _recording_client(**response_kwargs):
return requests, HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle)))
def _async_recording_client(**response_kwargs):
requests: list[httpx.Request] = []
async def handle(request):
requests.append(request)
return httpx.Response(200, **response_kwargs)
return requests, AsyncHTTPHandler(transport=httpx.MockTransport(handle))
@pytest.mark.parametrize(
"model, model_path",
[
@ -550,20 +562,106 @@ def test_bearer_api_key_is_sent_as_the_authorization_header(monkeypatch):
assert headers["Authorization"] == "Bearer bedrock-api-key"
FILE_SEARCH_TOOL = {"type": "file_search", "vector_store_ids": ["vs_other_team"]}
def _weather_tool_named(name):
return {**GET_WEATHER_TOOL, "function": {**GET_WEATHER_TOOL["function"], "name": name}}
@pytest.mark.parametrize(
"request_params, expected_route",
[
({"tools": [GET_WEATHER_TOOL]}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": None}, "converse"),
({"tools": [GET_WEATHER_TOOL]}, "responses"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}, "responses"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": None}, "responses"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "none"}, "chat_completions"),
(
{
"tools": [GET_WEATHER_TOOL],
"reasoning_effort": "low",
"guardrailConfig": {"guardrailIdentifier": "gr-1"},
},
"converse",
),
({"tools": [GET_WEATHER_TOOL, FILE_SEARCH_TOOL], "reasoning_effort": "low"}, "converse"),
({"tools": GET_WEATHER_TOOL, "reasoning_effort": "low"}, "converse"),
({"tools": ["get_weather"], "reasoning_effort": "low"}, "converse"),
({"tools": [{"type": "function"}], "reasoning_effort": "low"}, "converse"),
({"tools": [_weather_tool_named("get weather!")], "reasoning_effort": "low"}, "converse"),
({"tools": [_weather_tool_named("weather.get")], "reasoning_effort": "low"}, "converse"),
({"tools": [_weather_tool_named("w" * 65)], "reasoning_effort": "low"}, "converse"),
({"tools": [_weather_tool_named("w" * 64)], "reasoning_effort": "low"}, "responses"),
({"tools": [_weather_tool_named("get-weather_2")], "reasoning_effort": "low"}, "responses"),
(
{"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"tools": [FILE_SEARCH_TOOL]}},
"converse",
),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"store": False}}, "responses"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"store": None}}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"store": "false"}}, "converse"),
(
{"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"previous_response_id": "resp_1"}},
"converse",
),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "previous_response_id": "resp_1"}, "converse"),
(
{"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"instructions": "Be terse."}},
"converse",
),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "instructions": "Be terse."}, "converse"),
(
{"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"prompt": {"id": "pmpt_1"}}},
"converse",
),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"top_k": 5}}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": "store=false"}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "include": ["file_search_call.results"]}, "responses"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "include": "file_search_call.results"}, "converse"),
(
{"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"include": ["file_search_call.results"]}},
"responses",
),
(
{"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"include": "file_search_call.results"}},
"converse",
),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"include": {"a": 1}}}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "seed": 7}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "logit_bias": {"50256": -100}}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "web_search_options": {}}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "n": 2}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "n": 1}, "responses"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "presence_penalty": 0.5}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "presence_penalty": 0}, "responses"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "logprobs": True}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "logprobs": False}, "responses"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "prompt_cache_key": "k"}, "responses"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": {"effort": "low", "summary": "auto"}}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "store": "false"}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "store": 0}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "foo_bar": 1}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "foo_bar": None}, "responses"),
(
{
"tools": [GET_WEATHER_TOOL],
"reasoning_effort": "low",
"model": "global.openai.gpt-6-sol",
"messages": [{"role": "user", "content": "hi"}],
"drop_params": True,
"aws_region_name": "us-east-1",
"metadata": {"team": "a"},
"reasoning_summary": "auto",
},
"responses",
),
({"reasoning_effort": "low"}, "chat_completions"),
({"tools": None, "reasoning_effort": "low"}, "chat_completions"),
({"tools": [], "reasoning_effort": "low"}, "chat_completions"),
({}, "chat_completions"),
],
)
def test_gpt56_tools_need_reasoning_none_on_chat_completions(local_cost_map, request_params, expected_route):
def test_gpt56_tools_with_reasoning_bridge_to_responses(local_cost_map, request_params, expected_route):
assert (
BedrockModelInfo.get_bedrock_route("chat_completions/global.openai.gpt-5.6-sol", request_params)
== expected_route
@ -604,6 +702,21 @@ def test_gpt56_legacy_functions_route_like_tools(local_cost_map, request_params,
)
def test_gpt56_tools_with_reasoning_stay_on_converse_without_a_responses_endpoint(local_cost_map, monkeypatch):
model = "global.openai.gpt-6-sol"
monkeypatch.setitem(
litellm.model_cost, model, {**litellm.model_cost[model], "supported_endpoints": ["/v1/chat/completions"]}
)
params = {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}
assert BedrockModelInfo.get_bedrock_route(model, params) == "converse"
assert BedrockModelInfo.get_bedrock_route(model, {**params, "reasoning_effort": "none"}) == "chat_completions"
def test_gpt56_region_path_tools_with_reasoning_stay_on_converse(local_cost_map):
params = {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}
assert BedrockModelInfo.get_bedrock_route("bedrock/us-west-2/global.openai.gpt-6-sol", params) == "converse"
def test_thinking_block_goes_to_converse(local_cost_map):
thinking = {"type": "enabled", "budget_tokens": 1024}
assert BedrockModelInfo.get_bedrock_route("chat_completions/us.xai.grok-4.6", {"thinking": thinking}) == "converse"
@ -1075,19 +1188,481 @@ def test_gpt_oss_completion_hits_chat_completions_and_splits_reasoning(local_cos
assert response.choices[0].message.content == "Hi"
def test_gpt56_tools_with_reasoning_effort_go_to_converse(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=CONVERSE_JSON)
response = litellm.completion(
model="bedrock/chat_completions/global.openai.gpt-5.6-sol",
messages=[{"role": "user", "content": "hello"}],
RESPONSES_TOOL_CALL_JSON = {
"id": "resp_test",
"object": "response",
"created_at": 1733529600,
"status": "completed",
"output": [
{
"type": "function_call",
"id": "fc_0",
"call_id": "call_0",
"name": "get_weather",
"arguments": '{"city": "Paris"}',
"status": "completed",
}
],
"usage": {"input_tokens": 5, "output_tokens": 3, "total_tokens": 8},
}
@pytest.mark.parametrize(
"model", ["bedrock/chat_completions/global.openai.gpt-5.6-sol", "bedrock/global.openai.gpt-6-sol"]
)
async def test_gpt56_and_newer_tools_with_reasoning_effort_bridge_to_runtime_responses(
local_cost_map, fake_aws_env, model
):
requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": model})
response = await litellm.acompletion(
model=model,
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
temperature=0.2,
drop_params=True,
client=client,
)
assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/responses"
body = json.loads(requests[0].content)
assert body["model"] == model.removeprefix("bedrock/").removeprefix("chat_completions/")
assert body["reasoning"] == {"effort": "low"}
assert [tool["name"] for tool in body["tools"]] == ["get_weather"]
assert "temperature" not in body
assert body["store"] is False
assert body["include"] == ["reasoning.encrypted_content"]
assert response.object == "chat.completion"
assert response.choices[0].finish_reason == "tool_calls"
assert response.choices[0].message.tool_calls[0].function.name == "get_weather"
assert response._hidden_params["response_cost"] > 0
async def test_a_provider_specific_param_keeps_converse_and_reaches_the_model(local_cost_map, fake_aws_env):
requests, client = _async_recording_client(json=CONVERSE_JSON)
await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
foo_bar=1,
client=client,
)
assert [request.url.raw_path for request in requests] == [b"/model/global.openai.gpt-6-sol/converse"]
assert json.loads(requests[0].content)["additionalModelRequestFields"]["foo_bar"] == 1
async def test_tools_with_reasoning_bridge_stores_only_when_the_caller_asks(local_cost_map, fake_aws_env):
requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"})
await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
store=True,
client=client,
)
body = json.loads(requests[0].content)
assert body["store"] is True
assert "include" not in body
@pytest.mark.parametrize(
"include_param",
[
{"include": ["file_search_call.results"]},
{"extra_body": {"include": ["file_search_call.results"]}},
],
)
async def test_tools_with_reasoning_bridge_keeps_encrypted_reasoning_beside_a_requested_include(
local_cost_map, fake_aws_env, include_param
):
requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"})
await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
client=client,
**include_param,
)
body = json.loads(requests[0].content)
assert body["store"] is False
assert body["include"] == ["file_search_call.results", "reasoning.encrypted_content"]
async def test_tools_with_reasoning_bridge_sends_a_tool_reference_result_as_its_tool_name_like_converse(
local_cost_map, fake_aws_env
):
requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"})
await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[
{"role": "user", "content": "find a web tool"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "ToolSearch", "arguments": "{}"}}
],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": [{"type": "tool_reference", "tool_name": "WebSearch"}],
},
],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
client=client,
)
assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-5.6-sol/converse")
assert json.loads(requests[0].content)["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather"
assert response.choices[0].message.content == "ok"
body = json.loads(requests[0].content)
tool_output = next(item for item in body["input"] if item["type"] == "function_call_output")
assert tool_output["output"] == [{"type": "input_text", "text": "WebSearch"}]
@pytest.mark.parametrize("stream", [False, True], ids=["plain", "streamed"])
async def test_tools_with_reasoning_bridge_returns_the_provider_response_headers(local_cost_map, fake_aws_env, stream):
reply_kwargs: Final = (
{
"content": _responses_stream_with_reasoning("bedrock-minted", "global.openai.gpt-6-sol"),
"headers": {"content-type": "text/event-stream", "x-amzn-requestid": "req-1"},
}
if stream
else {
"json": _responses_reply_with_reasoning("bedrock-minted", "global.openai.gpt-6-sol"),
"headers": {"x-amzn-requestid": "req-1"},
}
)
_, response = await _bedrock_turn([{"role": "user", "content": "weather in Paris"}], **reply_kwargs)
assert response._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-1"
async def test_tools_with_reasoning_and_a_param_the_bridge_drops_fail_like_converse_without_drop_params(
local_cost_map, fake_aws_env
):
requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"})
with pytest.raises(litellm.UnsupportedParamsError, match="seed"):
await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
seed=7,
client=client,
)
assert requests == []
@pytest.mark.parametrize(
"runtime_endpoint_param, runtime_endpoint_env, expected_host",
[
("https://vpce-param.example.com", None, "vpce-param.example.com"),
(None, "https://vpce-env.example.com", "vpce-env.example.com"),
(None, None, "proxy.example.com"),
],
)
async def test_tools_with_reasoning_bridge_goes_to_the_host_converse_would(
local_cost_map, fake_aws_env, monkeypatch, runtime_endpoint_param, runtime_endpoint_env, expected_host
):
if runtime_endpoint_env:
monkeypatch.setenv("AWS_BEDROCK_RUNTIME_ENDPOINT", runtime_endpoint_env)
params = {
"model": "bedrock/global.openai.gpt-6-sol",
"messages": [{"role": "user", "content": "weather in Paris"}],
"reasoning_effort": "low",
"api_base": "https://proxy.example.com",
"aws_bedrock_runtime_endpoint": runtime_endpoint_param,
}
converse_requests, converse_client = _async_recording_client(json=CONVERSE_JSON)
await litellm.acompletion(**params, tools=[_weather_tool_named("get weather!")], client=converse_client)
bridge_requests, bridge_client = _async_recording_client(
json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}
)
await litellm.acompletion(**params, tools=[GET_WEATHER_TOOL], client=bridge_client)
assert converse_requests[0].url.raw_path.endswith(b"/converse")
assert bridge_requests[0].url.path == "/openai/v1/responses"
assert bridge_requests[0].url.host == converse_requests[0].url.host == expected_host
async def test_tools_with_reasoning_and_a_name_openai_rejects_keep_converse_and_its_name_mapping(
local_cost_map, fake_aws_env
):
requests, client = _async_recording_client(json=CONVERSE_JSON)
await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[_weather_tool_named("get weather!")],
reasoning_effort="low",
client=client,
)
assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse")
assert json.loads(requests[0].content)["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather_"
async def test_a_caller_param_named_like_the_bridge_flag_neither_crashes_nor_steers_the_route(
local_cost_map, fake_aws_env
):
requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"})
response = await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
_litellm_responses_api_bridge_allowed=False,
client=client,
)
assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/responses"
assert response.choices[0].message.tool_calls[0].function.name == "get_weather"
async def test_tools_with_reasoning_bridge_keeps_the_reasoning_summary_alias(local_cost_map, fake_aws_env):
requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"})
await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
extra_body={"reasoningSummary": "auto"},
client=client,
)
assert json.loads(requests[0].content)["reasoning"] == {"effort": "low", "summary": "auto"}
def test_sync_tools_with_reasoning_stay_on_converse(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=CONVERSE_JSON)
litellm.completion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
client=client,
)
assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse")
async def test_tools_with_an_object_reasoning_effort_stay_on_converse_like_main(local_cost_map, fake_aws_env):
requests, client = _async_recording_client(json=CONVERSE_JSON)
await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort={"effort": "low", "summary": "auto"},
client=client,
)
assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse")
def _responses_reply_with_reasoning(encrypted_content: str, model: str) -> dict:
return {
**RESPONSES_TOOL_CALL_JSON,
"model": model,
"output": [
{"type": "reasoning", "id": "rs_0", "encrypted_content": encrypted_content, "summary": []},
*RESPONSES_TOOL_CALL_JSON["output"],
],
}
def _responses_stream_with_reasoning(encrypted_content: str, model: str) -> bytes:
completed: Final = {"type": "response.completed", "response": _responses_reply_with_reasoning(encrypted_content, model)}
return f"event: response.completed\ndata: {json.dumps(completed)}\n\n".encode()
async def _bedrock_turn(
history: list, model: str = "global.openai.gpt-6-sol", **response_kwargs
) -> tuple[list[httpx.Request], object]:
requests, client = _async_recording_client(**response_kwargs)
response = await litellm.acompletion(
model=f"bedrock/{model}",
messages=history,
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
stream="text/event-stream" in response_kwargs.get("headers", {}).values(),
client=client,
)
return requests, response
async def _openai_turn(history: list, reply: dict) -> tuple[list[httpx.Request], object]:
requests, client = _async_recording_client(json=reply)
response = await litellm.acompletion(
model="openai/gpt-5.5-pro",
api_key="x",
api_base="http://127.0.0.1:9",
messages=history,
tools=[GET_WEATHER_TOOL],
client=client,
)
return requests, response
def _history_after(reasoning_items: Iterable[Mapping[str, object]]) -> list[dict[str, object]]:
return [
{"role": "user", "content": "weather in Paris"},
{
"role": "assistant",
"content": None,
"reasoning_items": [dict(item) for item in reasoning_items],
"tool_calls": [
{
"id": "call_0",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city": "Paris"}'},
}
],
},
{"role": "tool", "tool_call_id": "call_0", "content": "14C, light rain"},
]
def _replayed_reasoning(request: httpx.Request) -> list:
return [
item.get("encrypted_content") for item in json.loads(request.content)["input"] if item.get("type") == "reasoning"
]
async def _streamed_reasoning_items(response) -> list:
deltas: Final = [chunk.choices[0].delta async for chunk in response if chunk.choices]
return [item for delta in deltas for item in getattr(delta, "reasoning_items", None) or ()]
@pytest.mark.parametrize("stream", [False, True], ids=["plain", "streamed"])
async def test_reasoning_bedrock_minted_replays_to_bedrock_but_never_to_openai(local_cost_map, fake_aws_env, stream):
reply_kwargs: Final = (
{
"content": _responses_stream_with_reasoning("bedrock-minted", "global.openai.gpt-6-sol"),
"headers": {"content-type": "text/event-stream"},
}
if stream
else {"json": _responses_reply_with_reasoning("bedrock-minted", "global.openai.gpt-6-sol")}
)
_, first = await _bedrock_turn([{"role": "user", "content": "weather in Paris"}], **reply_kwargs)
reasoning_items: Final = (
await _streamed_reasoning_items(first) if stream else list(first.choices[0].message.reasoning_items)
)
history: Final = _history_after(reasoning_items)
to_bedrock, _ = await _bedrock_turn(history, json=RESPONSES_TOOL_CALL_JSON)
to_openai, _ = await _openai_turn(history, {**RESPONSES_TOOL_CALL_JSON, "model": "gpt-5.5-pro"})
assert len(reasoning_items) == 1
assert _replayed_reasoning(to_bedrock[0]) == ["bedrock-minted"]
assert _replayed_reasoning(to_openai[0]) == []
async def test_reasoning_one_bedrock_model_minted_never_replays_to_another(local_cost_map, fake_aws_env):
_, first = await _bedrock_turn(
[{"role": "user", "content": "weather in Paris"}],
json=_responses_reply_with_reasoning("gpt-6-minted", "global.openai.gpt-6-sol"),
)
history: Final = _history_after(first.choices[0].message.reasoning_items)
to_same_model, _ = await _bedrock_turn(history, json=RESPONSES_TOOL_CALL_JSON)
to_same_model_us_profile, _ = await _bedrock_turn(
history, model="us.openai.gpt-6-sol", json=RESPONSES_TOOL_CALL_JSON
)
to_other_model, _ = await _bedrock_turn(history, model="us.openai.gpt-5.6-sol", json=RESPONSES_TOOL_CALL_JSON)
assert _replayed_reasoning(to_same_model[0]) == ["gpt-6-minted"]
assert _replayed_reasoning(to_same_model_us_profile[0]) == ["gpt-6-minted"]
assert _replayed_reasoning(to_other_model[0]) == []
async def test_reasoning_openai_minted_replays_to_openai_but_never_to_bedrock(local_cost_map, fake_aws_env):
_, first = await _openai_turn(
[{"role": "user", "content": "weather in Paris"}], _responses_reply_with_reasoning("openai-minted", "gpt-5.5-pro")
)
history: Final = _history_after(first.choices[0].message.reasoning_items)
to_openai, _ = await _openai_turn(history, {**RESPONSES_TOOL_CALL_JSON, "model": "gpt-5.5-pro"})
to_bedrock, _ = await _bedrock_turn(history, json=RESPONSES_TOOL_CALL_JSON)
assert _replayed_reasoning(to_openai[0]) == ["openai-minted"]
assert _replayed_reasoning(to_bedrock[0]) == []
async def test_async_tools_with_reasoning_bridge_uses_the_supplied_client(local_cost_map, fake_aws_env):
requests: Final[list[httpx.Request]] = []
async def handle(request: httpx.Request) -> httpx.Response:
requests.append(request)
return httpx.Response(200, json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"})
response = await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
client=AsyncHTTPHandler(transport=httpx.MockTransport(handle)),
)
assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/responses"
assert response.choices[0].message.tool_calls[0].function.name == "get_weather"
class _FrozenCredentials:
access_key = "AKIDLEGACYCLIENT"
secret_key = "legacy-secret"
token = None
class _LegacyCredentials:
def get_frozen_credentials(self):
return _FrozenCredentials()
class _LegacyBedrockClient:
class meta:
region_name = "eu-west-1"
def _get_credentials(self):
return _LegacyCredentials()
async def test_tools_with_reasoning_and_a_legacy_aws_bedrock_client_stay_on_converse(local_cost_map, fake_aws_env):
requests, client = _async_recording_client(json=CONVERSE_JSON)
await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
aws_bedrock_client=_LegacyBedrockClient(),
client=client,
)
assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse")
assert requests[0].url.host == "bedrock-runtime.eu-west-1.amazonaws.com"
assert "Credential=AKIDLEGACYCLIENT/" in requests[0].headers["Authorization"]
def test_responses_forced_onto_chat_completions_serves_tools_with_reasoning_on_converse(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=CONVERSE_JSON)
litellm.responses(
model="bedrock/global.openai.gpt-6-sol",
input="weather in Paris",
tools=[{"type": "function", "name": "get_weather", "parameters": GET_WEATHER_TOOL["function"]["parameters"]}],
reasoning={"effort": "low"},
use_chat_completions_api=True,
client=client,
)
assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse")
body = json.loads(requests[0].content)
assert body["additionalModelRequestFields"] == {"reasoning": {"effort": "low"}}
assert body["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather"
def test_gpt56_tools_with_reasoning_none_stay_on_chat_completions(local_cost_map, fake_aws_env):
@ -1133,23 +1708,6 @@ def test_gpt_56_and_newer_completion_without_the_prefix_posts_runtime_chat_compl
assert response._hidden_params["response_cost"] > 0
def test_gpt6_without_the_prefix_tools_with_reasoning_effort_go_to_converse(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=CONVERSE_JSON)
response = litellm.completion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "hello"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
client=client,
)
assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse")
body = json.loads(requests[0].content)
assert body["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather"
assert body["additionalModelRequestFields"]["reasoning"] == {"effort": "low"}
assert response.choices[0].message.content == "ok"
def test_gpt6_without_the_prefix_guardrail_config_goes_to_converse(local_cost_map, fake_aws_env):
guardrail = {"guardrailIdentifier": "gr-1", "guardrailVersion": "1"}
requests, client = _recording_client(json=CONVERSE_JSON)

13
tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py
View file

@ -61,6 +61,19 @@ class TestCompleteURL:
)
assert url == "https://vpce.example.com/openai/v1/responses"
@pytest.mark.parametrize(
"runtime_endpoint_param, runtime_endpoint_env",
[("https://vpce.example.com", None), (None, "https://env-vpce.example.com")],
)
def test_api_base_wins_over_a_runtime_endpoint(self, monkeypatch, runtime_endpoint_param, runtime_endpoint_env):
if runtime_endpoint_env:
monkeypatch.setenv("AWS_BEDROCK_RUNTIME_ENDPOINT", runtime_endpoint_env)
url = _cfg().get_complete_url(
"https://proxy.example.com",
{"aws_region_name": "us-east-1", "aws_bedrock_runtime_endpoint": runtime_endpoint_param},
)
assert url == "https://proxy.example.com/openai/v1/responses"
class TestAuth:
def test_bearer_token_is_used_when_present(self):

32
ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts
View file

@ -335,6 +335,38 @@ describe("ToolsSection utils", () => {
expect(result[0].called).toBe(true);
expect(result[0].callData?.arguments).toEqual({ location: "Tokyo" });
});
it("should parse tool calls from a Responses API response output", () => {
const log: Partial<LogEntry> = {
request_id: "test-responses-1",
proxy_server_request: {
tools: [
{ type: "function", function: { name: "get_weather", description: "Get current weather" } },
{ type: "function", function: { name: "get_time", description: "Get current time" } },
],
},
response: {
object: "response",
output: [
{ type: "reasoning", summary: [] },
{
type: "function_call",
call_id: "call_abc",
name: "get_weather",
arguments: '{"location": "Paris"}',
},
],
},
} as any;
const result = parseToolsFromLog(log as LogEntry);
expect(result.map((tool) => [tool.name, tool.called])).toEqual([
["get_weather", true],
["get_time", false],
]);
expect(result[0].callData).toEqual({ id: "call_abc", name: "get_weather", arguments: { location: "Paris" } });
});
});
describe("hasTools", () => {

38
ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts
View file

@ -81,31 +81,35 @@ function extractToolCallsFromResponse(log: LogEntry): ToolCall[] {
return responseData.tool_calls;
}
if (Array.isArray(responseData.output)) {
const toolCalls = toolCallsFromOutput(responseData.output);
if (toolCalls.length > 0) return toolCalls;
}
// Realtime API format: response.results[].response.output[].type === "function_call"
if (Array.isArray(responseData.results)) {
const toolCalls: ToolCall[] = [];
for (const result of responseData.results) {
if (result.type === "response.done" && result.response?.output) {
for (const item of result.response.output) {
if (item.type === "function_call") {
toolCalls.push({
id: item.call_id || "",
type: "function",
function: {
name: item.name || "",
arguments: item.arguments || "{}",
},
});
}
}
}
}
const toolCalls = responseData.results
.filter((result: any) => result.type === "response.done" && Array.isArray(result.response?.output))
.flatMap((result: any) => toolCallsFromOutput(result.response.output));
if (toolCalls.length > 0) return toolCalls;
}
return [];
}
function toolCallsFromOutput(output: any[]): ToolCall[] {
return output
.filter((item: any) => item.type === "function_call")
.map((item: any) => ({
id: item.call_id || "",
type: "function",
function: {
name: item.name || "",
arguments: item.arguments || "{}",
},
}));
}
/**
* Parse safe JSON with fallback
*/