scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-24 05:45:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 81

Merge 7808c848f6 into 571ef3e2ae

Browse source
This commit is contained in:
Matthew Lapointe 2026-10-17 11:02:18 -07:00 • committed by GitHub
commit ca58cf1929
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 1222 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
litellm/completion_extras/litellm_responses_transformation/handler.py
View file

@ -11,7 +11,7 @@ from typing import (
cast, # noqa: TID251 # optional_params reaches the bridge as an untyped dict
)
from typing_extensions import TypedDict
from typing_extensions import ReadOnly, TypedDict
from litellm.types.llms.openai import ResponsesAPIResponse
@ -29,6 +29,7 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict):
logging_obj: "LiteLLMLoggingObj"
custom_llm_provider: str
encoding: object
client: ReadOnly[object]
def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:
@ -158,6 +159,7 @@ class ResponsesToCompletionBridgeHandler:
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
encoding=typed_kwargs.get("encoding"),
client=typed_kwargs.get("client"),
)
def completion(
@ -192,7 +194,7 @@ class ResponsesToCompletionBridgeHandler:
litellm_params=litellm_params,
headers=headers,
litellm_logging_obj=logging_obj,
client=kwargs.get("client"),
client=validated_kwargs["client"],
)
# Set on request_data rather than passed as explicit kwargs: the spread of
@ -251,7 +253,7 @@ class ResponsesToCompletionBridgeHandler:
else:
if self._is_preformatted_cached_chat_stream(result):
return self._apply_post_stream_processing(result, model, custom_llm_provider)
from .transformation import is_structured_output_request
from .transformation import is_structured_output_request, reasoning_origin_for
completion_stream: Final = self.transformation_handler.get_model_response_iterator(
streaming_response=result,
@ -260,6 +262,7 @@ class ResponsesToCompletionBridgeHandler:
structured_output=is_structured_output_request(
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
),
reasoning_origin=reasoning_origin_for(custom_llm_provider, model),
)
streamwrapper: Final = CustomStreamWrapper(
completion_stream=completion_stream,
@ -293,6 +296,7 @@ class ResponsesToCompletionBridgeHandler:
litellm_params=litellm_params,
headers=headers,
litellm_logging_obj=logging_obj,
client=validated_kwargs["client"],
)
except Exception as e:
raise e
@ -354,7 +358,7 @@ class ResponsesToCompletionBridgeHandler:
else:
if self._is_preformatted_cached_chat_stream(result):
return self._apply_post_stream_processing(result, model, custom_llm_provider)
from .transformation import is_structured_output_request
from .transformation import is_structured_output_request, reasoning_origin_for
completion_stream: Final = self.transformation_handler.get_model_response_iterator(
streaming_response=result,
@ -363,6 +367,7 @@ class ResponsesToCompletionBridgeHandler:
structured_output=is_structured_output_request(
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
),
reasoning_origin=reasoning_origin_for(custom_llm_provider, model),
)
streamwrapper: Final = CustomStreamWrapper(
completion_stream=completion_stream,

96
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -5,6 +5,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req
import json
import os
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from itertools import accumulate, chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, Union, cast
@ -305,6 +306,59 @@ def _reasoning_items_from_output_items(output_items: Sequence[object]) -> tuple[
)
_REASONING_ORIGIN_PREFIX: Final = "litellm_reasoning_origin:"
@dataclass(frozen=True, slots=True)
class ReasoningReplayTarget:
origin: str | None
def reasoning_origin_for(provider: str | None, model: str) -> str | None:
if provider != "bedrock":
return None
from litellm.llms.bedrock.common_utils import get_bedrock_base_model
return f"{_REASONING_ORIGIN_PREFIX}bedrock/{get_bedrock_base_model(model)}:"
def _with_reasoning_origin(output_items: Sequence[object], origin: str | None) -> Sequence[object]:
if origin is None:
return output_items
return tuple(_reasoning_output_item_with_origin(item, origin) for item in output_items)
def _reasoning_output_item_with_origin(item: object, origin: str) -> object:
from openai.types.responses import ResponseReasoningItem
if isinstance(item, ResponseReasoningItem):
return (
item.model_copy(update={"encrypted_content": f"{origin}{item.encrypted_content}"})
if item.encrypted_content
else item
)
if not isinstance(item, dict):
return item
fields: Final = cast(dict[str, object], item) # cast-ok: provider JSON item, keys forwarded verbatim
encrypted_content: Final = fields.get("encrypted_content")
if fields.get("type") != "reasoning" or not isinstance(encrypted_content, str) or not encrypted_content:
return fields
return {**fields, "encrypted_content": f"{origin}{encrypted_content}"}
def _replayable_reasoning_item(
item: Mapping[str, object], target: ReasoningReplayTarget
) -> Mapping[str, object] | None:
encrypted_content: Final = item.get("encrypted_content")
if not isinstance(encrypted_content, str) or not encrypted_content:
return item
if target.origin is None:
return None if encrypted_content.startswith(_REASONING_ORIGIN_PREFIX) else item
if not encrypted_content.startswith(target.origin):
return None
return {**item, "encrypted_content": encrypted_content.removeprefix(target.origin)}
_TOOL_SEARCH_ITEM_TYPES: Final = frozenset({"tool_search_call", "tool_search_output"})
_TOOL_SEARCH_REPLAY_ITEM_TYPES: Final = frozenset({"reasoning", *_TOOL_SEARCH_ITEM_TYPES})
_TOOL_SEARCH_ITEMS_KEY: Final = "tool_search_items"
@ -428,11 +482,18 @@ def _reasoning_ids(items: Sequence[Mapping[str, object]]) -> frozenset[str]:
return frozenset(item_id for item_id in ids if isinstance(item_id, str))
def _assistant_replay_items(msg: "AllMessageValues", replay_tool_search: bool) -> tuple[Mapping[str, object], ...]:
def _assistant_replay_items(
msg: "AllMessageValues", replay_tool_search: bool, replay_to: ReasoningReplayTarget | None
) -> tuple[Mapping[str, object], ...]:
searched: Final = _tool_search_input_items(msg) if replay_tool_search else ()
searched_reasoning_ids: Final = _reasoning_ids(searched)
reasoning: Final = (item for item in _reasoning_input_items(msg) if item.get("id") not in searched_reasoning_ids)
return (*searched, *reasoning)
replayable: Final = (
reasoning
if replay_to is None
else (kept for item in reasoning if (kept := _replayable_reasoning_item(item, replay_to)) is not None)
)
return (*searched, *replayable)
def _as_chat_reasoning_items(
@ -790,12 +851,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
keep_prompt_cache_breakpoints: bool = False,
replay_tool_search: bool = False,
include_namespace: bool = False,
replay_to: ReasoningReplayTarget | None = None,
) -> tuple[list[object], str | None]:
converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input(
messages,
drop_params=drop_params,
replay_tool_search=replay_tool_search,
include_namespace=include_namespace,
replay_to=replay_to,
)
return (
converted_input_items
@ -811,6 +874,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
drop_params: bool = False,
replay_tool_search: bool = False,
include_namespace: bool = False,
replay_to: ReasoningReplayTarget | None = None,
) -> tuple[list[object], str | None]:
input_items: Final[list[object]] = []
instructions: str | None = None
@ -892,7 +956,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
)
elif role == "assistant" and tool_calls and isinstance(tool_calls, list):
input_items.extend(_assistant_replay_items(msg, replay_tool_search))
input_items.extend(_assistant_replay_items(msg, replay_tool_search, replay_to))
if content:
input_items.append(
{
@ -933,7 +997,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
raise ValueError(f"tool call not supported: {tool_call}")
elif content is not None:
if role == "assistant":
input_items.extend(_assistant_replay_items(msg, replay_tool_search))
input_items.extend(_assistant_replay_items(msg, replay_tool_search, replay_to))
input_items.append(
{
"type": "message",
@ -944,7 +1008,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
}
)
elif role == "assistant":
input_items.extend(_assistant_replay_items(msg, replay_tool_search))
input_items.extend(_assistant_replay_items(msg, replay_tool_search, replay_to))
return input_items, instructions
@ -1065,6 +1129,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast
include_namespace=True,
replay_to=ReasoningReplayTarget(origin=reasoning_origin_for(provider, model)),
)
# OpenAI's Responses API rejects an empty input. For a system-only
# request, carry the system message as a system-role input item instead
@ -1419,8 +1484,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
# Convert response output to choices using the static helper
tagged_output_items: Final = _with_reasoning_origin(
output_items, reasoning_origin_for(_served_provider(litellm_params), model)
)
choices: Final = self._convert_response_output_to_choices(
output_items=output_items,
output_items=tagged_output_items,
handle_raw_dict_callback=self._handle_raw_dict_response_item,
structured_output=is_structured_output_request(
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
@ -1439,7 +1507,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
raw_response.incomplete_details.reason if raw_response.incomplete_details is not None else None
)
if len(choices) == 0:
choices.append(self._build_empty_choice(output_items, incomplete_finish_reason))
choices.append(self._build_empty_choice(tagged_output_items, incomplete_finish_reason))
else:
for choice in choices:
choice.finish_reason = incomplete_finish_reason
@ -1488,9 +1556,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
sync_stream: bool,
json_mode: bool | None = False,
structured_output: bool = False,
reasoning_origin: str | None = None,
) -> BaseModelResponseIterator:
return OpenAiResponsesToChatCompletionStreamIterator(
streaming_response, sync_stream, json_mode, structured_output=structured_output
streaming_response,
sync_stream,
json_mode,
structured_output=structured_output,
reasoning_origin=reasoning_origin,
)
def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]:
@ -1869,8 +1942,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
sync_stream: bool,
json_mode: bool | None = False,
structured_output: bool = False,
reasoning_origin: str | None = None,
):
super().__init__(streaming_response, sync_stream, json_mode)
self._reasoning_origin: Final = reasoning_origin
self._chat_completion_id: str | None = None
self._served_service_tier: str | None = None
self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state
@ -1916,6 +1991,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
def translate_responses_chunk_to_openai_stream(
parsed_chunk: dict | BaseModel,
tool_call_index_map: dict[int, int] | None = None, # mutable-ok: per-stream state, remapped in place
reasoning_origin: str | None = None,
) -> "ModelResponseStream":
"""
Translate a Responses API streaming chunk to OpenAI chat completion streaming format.
@ -2131,7 +2207,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
)
terminal_reasoning_items_typed: Final = _as_chat_reasoning_items(
_reasoning_items_from_output_items(output_items)
_reasoning_items_from_output_items(_with_reasoning_origin(output_items, reasoning_origin))
)
usage = None
@ -2195,7 +2271,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
self._empty_content_chunk()
if self._is_text_delta_after_structured_answer(chunk)
else OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(
chunk, tool_call_index_map=self._tool_call_index_map
chunk, tool_call_index_map=self._tool_call_index_map, reasoning_origin=self._reasoning_origin
)
)
if self._structured_output and _is_message_item_done(chunk):

154
litellm/llms/bedrock/common_utils.py
View file

@ -11,7 +11,7 @@ import os
import re
import warnings
from collections.abc import Iterator, Mapping, Sequence
from types import MappingProxyType
from types import MappingProxyType, NoneType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias
from typing_extensions import ReadOnly, TypedDict
@ -27,6 +27,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError
import litellm
from litellm import verbose_logger
from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES, OPENAI_CHAT_COMPLETION_PARAMS
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count
@ -38,6 +39,8 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.bedrock.request_metadata import bedrock_request_metadata_is_owned
from litellm.secret_managers.main import get_secret, get_secret_str
from litellm.types.llms.bedrock import AWS_AUTH_PARAM_KEYS, AwsAuthParams
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
from litellm.types.utils import is_litellm_owned_kwarg
if TYPE_CHECKING:
from litellm.llms.custom_httpx.http_handler import HTTPHandler
@ -61,6 +64,7 @@ BedrockRoute = Literal[
"openai",
"mantle",
"chat_completions",
"responses",
]
@ -1092,7 +1096,8 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje
written onto the Converse body,
function tools (``tools`` or legacy ``functions``) on a model without
``supports_bedrock_runtime_chat_completions_tools_with_reasoning`` are rejected there unless
``reasoning_effort`` is exactly ``"none"``, and a ``response_format`` goes native only as
``reasoning_effort`` is exactly ``"none"`` (function-only ``tools`` on a model listing ``/v1/responses`` are
bridged to bedrock-runtime's native Responses API instead), and a ``response_format`` goes native only as
``{"type": "json_schema", "json_schema": ...}`` (a pydantic model is converted to that) on a model with
``supports_bedrock_runtime_chat_completions_response_format``: a schema on any other model is only
honored by Converse, and every ``json_object`` form (``response_schema`` included) keeps Converse's
@ -1110,6 +1115,12 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje
return True
if _response_format_needs_converse(model, request_params.get("response_format")):
return True
return _chat_completions_rejects_function_tools(model, request_params) and not _bridgeable_to_responses(
model, request_params
)
def _chat_completions_rejects_function_tools(model: str, request_params: Mapping[str, object]) -> bool:
if not (request_params.get("tools") or request_params.get("functions")):
return False
return (
@ -1118,12 +1129,140 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje
)
class _NamedFunction(TypedDict):
name: ReadOnly[str]
class _FunctionTool(TypedDict):
type: ReadOnly[Literal["function"]]
function: ReadOnly[_NamedFunction]
_FUNCTION_TOOLS_ADAPTER: Final = TypeAdapter(tuple[_FunctionTool, ...])
_OPENAI_FUNCTION_NAME: Final = re.compile(r"[a-zA-Z0-9_-]{1,64}")
def _function_tools_only(tools: object) -> bool:
if not isinstance(tools, (list, tuple)):
return False
try:
validated: Final = _FUNCTION_TOOLS_ADAPTER.validate_python(tools)
except ValidationError:
return False
return all(_OPENAI_FUNCTION_NAME.fullmatch(tool["function"]["name"]) for tool in validated)
_EXTRA_BODY_ADAPTER: Final = TypeAdapter(dict[str, object])
_EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES: Final = frozenset(
{*ResponsesAPIOptionalRequestParams.__annotations__, "reasoning_effort", "reasoningSummary", "reasoning_summary"}
- {"tools"}
)
_CHAT_PARAMS_THE_RESPONSES_BRIDGE_DROPS: Final = MappingProxyType(
{
"n": 1,
"presence_penalty": 0,
"frequency_penalty": 0,
"logprobs": False,
"top_logprobs": None,
"logit_bias": None,
"seed": None,
"modalities": None,
"audio": None,
"prediction": None,
"web_search_options": None,
"function_call": None,
"include_server_side_tool_invocations": None,
}
)
_ENCRYPTED_REASONING: Final = "reasoning.encrypted_content"
_INCLUDE_ADAPTER: Final = TypeAdapter(tuple[object, ...])
def _extra_body_bridgeable(extra_body: object) -> bool:
if extra_body is None:
return True
return isinstance(extra_body, dict) and (
_EXTRA_BODY_ADAPTER.validate_python(extra_body).keys() <= _EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES
)
def _sets_a_chat_param_the_responses_bridge_drops(request_params: Mapping[str, object]) -> bool:
return any(
request_params.get(param) not in (None, no_op_value)
for param, no_op_value in _CHAT_PARAMS_THE_RESPONSES_BRIDGE_DROPS.items()
)
_CHAT_PARAM_NAMES: Final = frozenset(
{*DEFAULT_CHAT_COMPLETION_PARAM_VALUES, *OPENAI_CHAT_COMPLETION_PARAMS, "extra_body"}
)
def _provider_specific_param(name: str) -> bool:
return not (
name in _CHAT_PARAM_NAMES
or name in _EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES
or name.startswith("aws_")
or is_litellm_owned_kwarg(name)
)
def _passes_a_provider_specific_param(request_params: Mapping[str, object]) -> bool:
return any(value is not None and _provider_specific_param(name) for name, value in request_params.items())
def _bridgeable_to_responses(model: str, request_params: Mapping[str, object]) -> bool:
return (
not request_params.get("functions")
and request_params.get("aws_bedrock_client") is None
and _function_tools_only(request_params.get("tools"))
and _extra_body_bridgeable(request_params.get("extra_body"))
and not _passes_a_provider_specific_param(request_params)
and not _sets_a_chat_param_the_responses_bridge_drops(request_params)
and isinstance(request_params.get("reasoning_effort"), (str, NoneType))
and isinstance(request_params.get("store"), (bool, NoneType))
and _bedrock_serves_responses(model)
)
def _bedrock_serves_responses(model: str) -> bool:
region, model_id = split_bedrock_region_path(model)
return region is None and any(
_price_map_entry_lists_endpoint(_model_cost_entry(key), "/v1/responses")
for key in (model_id, f"bedrock/{model_id}")
)
def bedrock_responses_bridge_optional_params(optional_params: Mapping[str, object]) -> Mapping[str, object]:
if optional_params.get("store") is not None:
return optional_params
include: Final = optional_params.get("include")
requested: Final = _INCLUDE_ADAPTER.validate_python(include) if isinstance(include, (list, tuple)) else ()
return {
**optional_params,
"store": False,
"include": [*requested, *(() if _ENCRYPTED_REASONING in requested else (_ENCRYPTED_REASONING,))],
}
def bedrock_responses_bridge_api_base(api_base: str | None, aws_bedrock_runtime_endpoint: object) -> str | None:
return (
(aws_bedrock_runtime_endpoint if isinstance(aws_bedrock_runtime_endpoint, str) else None)
or get_secret_str("AWS_BEDROCK_RUNTIME_ENDPOINT")
or api_base
)
def _chat_completions_unless_converse_needed(
model: str, request_params: Mapping[str, object] | None
) -> Literal["converse", "chat_completions"]:
if request_params is not None and bedrock_request_needs_converse(model, request_params):
) -> Literal["converse", "chat_completions", "responses"]:
if request_params is None:
return "chat_completions"
if bedrock_request_needs_converse(model, request_params):
return "converse"
return "chat_completions"
if not _chat_completions_rejects_function_tools(model, request_params):
return "chat_completions"
return "converse" if request_params.get("_litellm_responses_api_bridge_allowed") is False else "responses"
def bedrock_route_for_request(
@ -1607,8 +1746,9 @@ class BedrockModelInfo(BaseLLMModelInfo):
Grok and GPT 5.6 and newer go to bedrock-runtime's native OpenAI Chat Completions by
default (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in;
``request_params`` (the caller's chat params) sends such a request to Converse when it needs a
feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family
model stays on Converse without the prefix.
feature only Converse serves, to ``responses`` (the chat-to-Responses bridge onto bedrock-runtime's
native ``/openai/v1/responses``) when it carries function tools with reasoning on, and ``converse/``
pins a model to Converse. Every other OpenAI-family model stays on Converse without the prefix.
"""
route_mappings: dict[
str,

48
litellm/main.py
View file

@ -119,6 +119,8 @@ from litellm.llms.base_llm.base_model_iterator import (
from litellm.llms.base_llm.chat.transformation import with_attribution_headers
from litellm.llms.bedrock.common_utils import (
BedrockModelInfo,
bedrock_responses_bridge_api_base,
bedrock_responses_bridge_optional_params,
bedrock_route_for_request,
without_bedrock_route_prefix,
)
@ -5689,6 +5691,7 @@ def completion(
"allowed_openai_params": allowed_openai_params,
"base_model": base_model,
}
bedrock_responses_bridge_allowed: Final = acompletion is True and not skip_responses_api_bridge
provider_reasoning_effort: Final = reasoning_effort_with_summary_alias(
reasoning_effort,
non_default_params,
@ -5699,6 +5702,7 @@ def completion(
**{**optional_param_args, "reasoning_effort": provider_reasoning_effort},
model_info=_MODEL_INFO_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else None,
**non_default_params,
_litellm_responses_api_bridge_allowed=bedrock_responses_bridge_allowed,
)
processed_non_default_params: Final = pre_process_non_default_params(
model=model,
@ -5863,14 +5867,39 @@ def completion(
api_base=api_base,
)
request_params: Final = MappingProxyType(
_OPTIONAL_PARAMS_ADAPTER.validate_python(
{
**optional_param_args,
**non_default_params,
"_litellm_responses_api_bridge_allowed": bedrock_responses_bridge_allowed,
}
)
)
bedrock_bridges_to_responses: Final = (
custom_llm_provider == "bedrock"
and bedrock_route_for_request(
model,
request_params,
_ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(kwargs.get("additional_drop_params") or ()),
)
== "responses"
)
# Use base_model (the true underlying model) for Azure model-type
# detection when the deployment name differs from the model name.
_azure_detection_model: Final = base_model or model
if responses_api_model_info.get("mode") == "responses" and not skip_responses_api_bridge:
if (
responses_api_model_info.get("mode") == "responses" or bedrock_bridges_to_responses
) and not skip_responses_api_bridge:
from litellm.completion_extras import responses_api_bridge
optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params)
optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(
{**bedrock_responses_bridge_optional_params(optional_params)}
if bedrock_bridges_to_responses
else optional_params
)
if isinstance(reasoning_effort, dict) and "summary" in reasoning_effort:
optional_params["reasoning_effort"] = reasoning_effort
@ -5896,7 +5925,16 @@ def completion(
acompletion=acompletion,
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
litellm_params=(
{
**litellm_params,
"api_base": bedrock_responses_bridge_api_base(
api_base, litellm_params.get("aws_bedrock_runtime_endpoint")
),
}
if bedrock_bridges_to_responses
else litellm_params
),
timeout=timeout,
client=client, # pass AsyncOpenAI, OpenAI client
custom_llm_provider=custom_llm_provider,
@ -5935,9 +5973,7 @@ def completion(
optional_params=optional_params,
organization=organization,
provider_config=provider_config,
request_params=MappingProxyType(
_OPTIONAL_PARAMS_ADAPTER.validate_python({**optional_param_args, **non_default_params})
),
request_params=request_params,
shared_session=shared_session,
stream=stream,
temperature=temperature,

8
litellm/utils.py
View file

@ -4576,6 +4576,7 @@ def get_optional_params(
prompt_cache_key: str | None = None,
base_model: str | None = None,
model_info: Mapping[str, object] | None = None,
_litellm_responses_api_bridge_allowed: bool = True,
**kwargs: object,
):
drop_params = normalize_drop_params(drop_params) # rebind-ok: config and DB deployments pass "true" as a string
@ -4587,6 +4588,7 @@ def get_optional_params(
# OpenAI param.
passed_params.pop("base_model", None)
passed_params.pop("model_info", None)
passed_params.pop("_litellm_responses_api_bridge_allowed")
provider_config: BaseConfig | None = None
if custom_llm_provider is not None and custom_llm_provider in [provider.value for provider in LlmProviders]:
provider_config = ProviderConfigManager.get_provider_chat_config(
@ -4638,7 +4640,11 @@ def get_optional_params(
)
bedrock_route: Final = (
_bedrock_route_for_request(model, passed_params, additional_drop_params)
_bedrock_route_for_request(
model,
{**passed_params, "_litellm_responses_api_bridge_allowed": _litellm_responses_api_bridge_allowed},
additional_drop_params,
)
if custom_llm_provider == "bedrock"
else None
)

90
tests/integration/messages_endpoint/providers/bedrock/test_bedrock_messages_gpt_chat_completions_wire.py
View file

@ -4,7 +4,15 @@ from collections.abc import Mapping
from typing import Final
import anthropic
from integration._support.bedrock_runtime_peer import NATIVE_CHAT, answer, body_of, marker_of, respond, target_of
from integration._support.bedrock_runtime_peer import (
NATIVE_CHAT,
NATIVE_RESPONSES,
answer,
body_of,
marker_of,
respond,
target_of,
)
from integration._support.client import Gateway, Scenario, eventually
from integration._support.database import read_rows
from integration._support.wire import Request, Wire, wire_server
@ -14,6 +22,7 @@ BEDROCK_MODEL: Final = "us.openai.gpt-5.6-sol"
TOKEN: Final = "synthetic-bedrock-bearer"
NO_CACHE: Final[Mapping[str, JsonValue]] = {"cache": {"no-cache": True}}
ANTHROPIC_VERSION: Final[Mapping[str, str]] = {"anthropic-version": "2023-06-01"}
TOOL_SCHEMA: Final[dict[str, object]] = {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}
def _question(marker: str) -> str:
@ -40,6 +49,41 @@ def _native_body(wire: Wire, marker: str) -> Mapping[str, JsonValue]:
return body_of(received[0])
def _bridged_body(wire: Wire, marker: str) -> Mapping[str, JsonValue]:
received: Final = _carrying(wire, marker)
assert [(request.method, target_of(request)) for request in received] == [("POST", NATIVE_RESPONSES)]
assert received[0].headers["authorization"] == f"Bearer {TOKEN}", received[0].headers
body: Final = body_of(received[0])
assert (body["model"], body["reasoning"], body["max_output_tokens"]) == (
BEDROCK_MODEL,
{"effort": "medium"},
4096,
), body
assert (body["store"], body["include"]) == (False, ["reasoning.encrypted_content"]), body
assert [(tool["type"], tool["name"], tool["parameters"]) for tool in _tools_of(body)] == [
("function", "lookup_invoice", TOOL_SCHEMA)
], body
return body
def _tools_of(body: Mapping[str, JsonValue]) -> list[dict[str, JsonValue]]:
tools: Final = body["tools"]
assert isinstance(tools, list), body
return [tool for tool in tools if isinstance(tool, dict)]
def _bridged_spend_row(model: str) -> dict[str, JsonValue]:
rows: Final = eventually(
lambda: read_rows(
'SELECT status, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE model_group=%s',
(model,),
),
lambda found: len(found) == 1,
seconds=70,
)
return rows[0]
def _native_request(marker: str, max_tokens: int, effort: str) -> Mapping[str, JsonValue]:
return {
"model": BEDROCK_MODEL,
@ -122,6 +166,50 @@ def test_anthropic_sdk_stream_with_thinking_budget_is_served_by_native_chat_comp
assert _spend_rows(identity, 1) == [_success_row(identity, model, cache_hit="False")]
def test_anthropic_sdk_tools_with_thinking_are_bridged_to_native_responses(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
client: Final = anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0)
message: Final = client.messages.create(
model=model,
max_tokens=4096,
thinking={"type": "enabled", "budget_tokens": 2048},
tools=[{"name": "lookup_invoice", "description": "Look up an invoice", "input_schema": TOOL_SCHEMA}],
messages=[{"role": "user", "content": _question(marker)}],
extra_body=NO_CACHE,
)
assert "stream" not in _bridged_body(wire, marker)
assert [(block.type, getattr(block, "text", None)) for block in message.content] == [("text", answer(marker))]
assert (message.usage.input_tokens, message.usage.output_tokens) == (30, 5), message
assert _bridged_spend_row(model) == {"status": "success", "prompt_tokens": 30, "completion_tokens": 5}
def test_anthropic_sdk_stream_with_tools_and_thinking_is_bridged_to_native_responses(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
client: Final = anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0)
stream: Final = client.messages.create(
model=model,
max_tokens=4096,
thinking={"type": "enabled", "budget_tokens": 2048},
tools=[{"name": "lookup_invoice", "description": "Look up an invoice", "input_schema": TOOL_SCHEMA}],
messages=[{"role": "user", "content": _question(marker)}],
extra_body=NO_CACHE,
stream=True,
)
events: Final = list(stream)
assert _bridged_body(wire, marker)["stream"] is True
assert events[0].type == "message_start" and events[-1].type == "message_stop", events
assert "".join(
event.delta.text
for event in events
if event.type == "content_block_delta" and event.delta.type == "text_delta"
) == answer(marker)
assert _bridged_spend_row(model) == {"status": "success", "prompt_tokens": 30, "completion_tokens": 5}
def test_raw_thinking_summary_reaches_native_chat_completions_as_the_plain_effort(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:

256
tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py
View file

@ -8,18 +8,21 @@ from urllib.parse import quote
import httpx
import openai
import pytest
from integration._support.bedrock_runtime_peer import answer, respond, target_of
from integration._support.client import Gateway, Scenario, eventually
from integration._support import responses_vendor as rv
from integration._support.bedrock_runtime_peer import NATIVE_RESPONSES, answer, respond, target_of
from integration._support.client import Gateway, Scenario, eventually, list_value, object_value
from integration._support.database import read_rows
from integration._support.sigv4 import signature
from integration._support.wire import Request, Wire, wire_server
from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
from openai.types.chat import ChatCompletionChunk, ChatCompletionFunctionToolParam, ChatCompletionMessageParam
from openai.types.chat.chat_completion_chunk import ChoiceDelta
from pydantic import JsonValue, TypeAdapter
GPT: Final = "us.openai.gpt-5.6-sol"
GLOBAL_GPT: Final = "global.openai.gpt-5.6-sol"
GPT_OSS: Final = "openai.gpt-oss-120b-1:0"
OTHER_GPT: Final = "global.openai.gpt-6.1-sol"
OPENAI_CODEX: Final = "gpt-5.3-codex"
TOKEN: Final = "synthetic-bedrock-bearer"
ACCESS_KEY: Final = "AKIASYNTHETICKEY0001"
SECRET_KEY: Final = "synthetic-secret-key-for-testing"
@ -45,6 +48,15 @@ TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType(
},
}
)
SDK_TOOL: Final[ChatCompletionFunctionToolParam] = {
"type": "function",
"function": {
"name": "lookup_invoice",
"description": "Look up an invoice",
"parameters": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]},
},
}
ENCRYPTED_REASONING: Final = "reasoning.encrypted_content"
CONVERSE_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType(
{
"toolSpec": {
@ -159,10 +171,27 @@ def _spend_row(identity: str) -> dict[str, JsonValue]:
return rows[0]
def _only_spend_row_of(model: str) -> dict[str, JsonValue]:
rows: Final = eventually(
lambda: read_rows(
'SELECT model_group, status, prompt_tokens, completion_tokens, api_base FROM "LiteLLM_SpendLogs"'
" WHERE model_group=%s",
(model,),
),
lambda found: len(found) == 1,
seconds=70,
)
return rows[0]
def _success_row(model: str, api_base: str) -> dict[str, JsonValue]:
return {"model_group": model, "status": "success", "prompt_tokens": 9, "completion_tokens": 5, "api_base": api_base}
def _bridged_success_row(model: str, wire_url: str) -> dict[str, JsonValue]:
return {**_success_row(model, f"{wire_url}{NATIVE_RESPONSES}"), "prompt_tokens": 30}
def _delta_text(delta: ChoiceDelta, field: str) -> str:
value: Final = delta.model_dump().get(field)
return value if isinstance(value, str) else ""
@ -387,17 +416,234 @@ def test_json_schema_response_format_is_forwarded_natively(gateway: Gateway) ->
assert _spend_row(f"chatcmpl-{marker}") == _success_row(model, f"{wire.url}{NATIVE_TARGET}")
def test_tools_while_reasoning_keep_converse(gateway: Gateway) -> None:
def test_tools_while_reasoning_are_bridged_to_native_responses(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high")
payload: Final = _payload(response)
assert payload["id"] == f"resp_upstream_{marker}", response.text
choices: Final = payload["choices"]
assert isinstance(choices, list) and len(choices) == 1, response.text
assert choices[0]["finish_reason"] == "stop", response.text
assert choices[0]["message"]["content"] == answer(marker), response.text
body: Final = _body(_converse_request(wire, NATIVE_RESPONSES))
assert body["model"] == GPT, body
assert body["reasoning"] == {"effort": "high"}, body
assert (body["store"], body["include"]) == (False, [ENCRYPTED_REASONING]), body
assert [(tool["type"], tool["name"], tool["parameters"]) for tool in body["tools"]] == [
("function", "lookup_invoice", dict(TOOL_PARAMETERS))
], body
assert _upstream_requests_mentioning(gateway, marker) == [], response.text
assert _only_spend_row_of(model) == _bridged_success_row(model, wire.url)
def test_streamed_tools_while_reasoning_are_bridged_to_native_responses(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
stream: Final = _openai_client(gateway).chat.completions.create(
model=model,
messages=_sdk_messages(marker),
tools=[SDK_TOOL],
reasoning_effort="high",
stream=True,
stream_options={"include_usage": True},
extra_body={"cache": {"no-cache": True}},
)
chunks: Final = list(stream)
assert _joined(chunks, "content") == answer(marker), chunks
usage: Final = chunks[-1].usage
assert usage is not None and (usage.prompt_tokens, usage.completion_tokens) == (30, 5), chunks[-1]
body: Final = _body(_converse_request(wire, NATIVE_RESPONSES))
assert (body["model"], body["stream"], body["reasoning"]) == (GPT, True, {"effort": "high"}), body
assert (body["store"], body["include"]) == (False, [ENCRYPTED_REASONING]), body
assert [tool["name"] for tool in body["tools"]] == ["lookup_invoice"], body
assert _only_spend_row_of(model) == _bridged_success_row(model, wire.url)
def _reasoning_deployment(scenario: Scenario, wire: Wire, target: str) -> str:
if target == OPENAI_CODEX:
return scenario.model(model=f"openai/{OPENAI_CODEX}", api_base=wire.url, api_key="synthetic-openai-key")
return _deployment(scenario, wire, model=f"bedrock/{target}")
def _message(payload: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
return object_value(object_value(list_value(payload["choices"])[0])["message"])
def _reasoning_turn(gateway: Gateway, model: str, marker: str) -> list[dict[str, JsonValue]]:
response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high")
return rv.ITEMS.validate_python(_message(_payload(response))["reasoning_items"])
def _replay(
gateway: Gateway, model: str, marker: str, reasoning_items: Sequence[Mapping[str, JsonValue]]
) -> dict[str, JsonValue]:
history: Final[list[JsonValue]] = [
{"role": "user", "content": "Pick an invoice."},
{"role": "assistant", "content": "Invoice 7", "reasoning_items": [dict(item) for item in reasoning_items]},
{"role": "user", "content": _prompt(marker)},
]
return _payload(
gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": history,
"tools": [dict(TOOL)],
"reasoning_effort": "high",
"cache": {"no-cache": True},
},
)
)
def _vendor_reasoning_item(item_id: str) -> dict[str, JsonValue]:
return {
"type": "reasoning",
"id": item_id,
"summary": [],
"encrypted_content": f"gAAAAA-vendor-{item_id.removeprefix('rs_')}",
}
@pytest.mark.parametrize("replay_to", [GLOBAL_GPT, GPT], ids=["to-another-inference-profile", "to-the-same-profile"])
def test_bridged_reasoning_replays_unchanged_to_the_bedrock_model_that_minted_it(
gateway: Gateway, replay_to: str
) -> None:
marker: Final = uuid.uuid4().hex
follow_up: Final = uuid.uuid4().hex
vendor: Final = rv.ResponsesVendor()
with wire_server(vendor.respond) as wire, gateway.scenario() as scenario:
minter: Final = _deployment(scenario, wire)
replayer: Final = minter if replay_to == GPT else _reasoning_deployment(scenario, wire, replay_to)
(item,) = _reasoning_turn(gateway, minter, marker)
(minted,) = vendor.minted
assert item["id"] == minted, item
assert target_of(_only_request(wire)) == NATIVE_RESPONSES
message: Final = _message(_replay(gateway, replayer, follow_up, [item]))
assert message["content"] == rv.answer(follow_up), message
body: Final = _body(_converse_request(wire, NATIVE_RESPONSES))
assert body["model"] == replay_to, body
assert rv.reasoning_items(body) == [_vendor_reasoning_item(minted)], body["input"]
@pytest.mark.parametrize(
("minted_on", "replayed_to"),
[(GPT, OTHER_GPT), (GPT, OPENAI_CODEX), (OPENAI_CODEX, GPT)],
ids=["bedrock-to-another-bedrock-model", "bedrock-to-openai", "openai-to-bedrock"],
)
def test_reasoning_is_not_replayed_to_a_model_that_did_not_mint_it(
gateway: Gateway, minted_on: str, replayed_to: str
) -> None:
marker: Final = uuid.uuid4().hex
follow_up: Final = uuid.uuid4().hex
vendor: Final = rv.ResponsesVendor()
with wire_server(vendor.respond) as wire, gateway.scenario() as scenario:
minter: Final = _reasoning_deployment(scenario, wire, minted_on)
replayer: Final = _reasoning_deployment(scenario, wire, replayed_to)
(item,) = _reasoning_turn(gateway, minter, marker)
assert item["id"] == vendor.minted[0], item
wire.drain()
message: Final = _message(_replay(gateway, replayer, follow_up, [item]))
assert message["content"] == rv.answer(follow_up), message
body: Final = _body(_only_request(wire))
assert body["model"] == replayed_to, body
assert rv.reasoning_items(body) == [], body["input"]
assert [entry.get("role") for entry in rv.ITEMS.validate_python(body["input"])] == [
"user",
"assistant",
"user",
], body["input"]
def test_tools_while_reasoning_with_a_provider_specific_field_keep_converse_and_reach_the_model(
gateway: Gateway,
) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high", foo_bar=1)
assert _message(_payload(response))["content"] == answer(marker), response.text
body: Final = _body(_converse_request(wire))
assert object_value(body["additionalModelRequestFields"])["foo_bar"] == 1, body
assert body["toolConfig"] == {"tools": [dict(CONVERSE_TOOL)]}, body
def test_tools_while_reasoning_with_a_non_boolean_store_are_refused_like_converse(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high", store="false")
assert response.status_code == 400, response.text
assert "store" in response.text, response.text
assert wire.drain() == (), response.text
def test_tools_while_reasoning_on_a_sigv4_deployment_are_bridged_signed_with_its_keys(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"bedrock/{GPT}",
api_key=None,
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
aws_region_name="us-east-1",
aws_bedrock_runtime_endpoint=wire.url,
)
response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high")
assert _payload(response)["id"] == f"resp_upstream_{marker}", response.text
request: Final = _only_request(wire)
assert (request.method, target_of(request)) == ("POST", NATIVE_RESPONSES), request.target
_assert_sigv4_signed(request, NATIVE_RESPONSES)
assert _body(request)["reasoning"] == {"effort": "high"}, request.body
assert _only_spend_row_of(model) == _bridged_success_row(model, wire.url)
@pytest.mark.parametrize("suffix", ["", "/openai/v1"])
def test_tools_while_reasoning_are_bridged_to_the_api_base_without_a_runtime_endpoint(
gateway: Gateway, suffix: str
) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model_info=None, **{**GPT_DEPLOYMENT, "api_base": f"{wire.url}{suffix}"})
response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high")
request: Final = _only_request(wire)
assert (request.method, request.target) == ("POST", NATIVE_RESPONSES), response.text
assert _payload(response)["id"] == f"resp_upstream_{marker}", response.text
assert _body(request)["reasoning"] == {"effort": "high"}, request.body
assert _only_spend_row_of(model) == _bridged_success_row(model, wire.url)
def test_tools_while_reasoning_effort_is_an_object_keep_converse(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(
gateway, model, marker, tools=[dict(TOOL)], reasoning_effort={"effort": "high", "summary": "auto"}
)
payload: Final = _payload(response)
assert payload["choices"][0]["message"]["content"] == answer(marker), response.text
assert _body(_converse_request(wire))["toolConfig"] == {"tools": [dict(CONVERSE_TOOL)]}
assert _spend_row(str(payload["id"])) == _success_row(model, f"{wire.url}{CONVERSE_TARGET}")
def test_tools_while_reasoning_with_a_name_openai_rejects_keep_converse(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
tool: Final = {**TOOL, "function": {**TOOL["function"], "name": "lookup invoice!"}}
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, tools=[tool], reasoning_effort="high")
payload: Final = _payload(response)
assert payload["choices"] == [
{"finish_reason": "stop", "index": 0, "message": {"content": answer(marker), "role": "assistant"}}
], response.text
body: Final = _body(_converse_request(wire))
assert body["toolConfig"] == {"tools": [CONVERSE_TOOL]}, body
assert body["toolConfig"] == {
"tools": [{"toolSpec": {**CONVERSE_TOOL["toolSpec"], "name": "lookup_invoice_"}}]
}, body
assert body["additionalModelRequestFields"] == {"reasoning": {"effort": "high"}}, body
assert _spend_row(str(payload["id"])) == _success_row(model, f"{wire.url}{CONVERSE_TARGET}")

61
tests/integration/sdk/test_bedrock_tools_while_reasoning_sync_sdk.py Normal file
View file

@ -0,0 +1,61 @@
import uuid
from collections.abc import Mapping
from typing import Final
import pytest
from integration._support.bedrock_runtime_peer import answer, respond, target_of
from integration._support.client import eventually
from integration._support.wire import wire_server
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.utils import Choices, ModelResponse
_MODEL_ID: Final = "us.openai.gpt-5.6-sol"
_TOOL: Final = {
"type": "function",
"function": {
"name": "lookup_invoice",
"description": "Look up an invoice",
"parameters": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]},
},
}
class _SuccessRecorder(CustomLogger):
successes: tuple[tuple[object, object], ...] = ()
def log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self.successes = (*self.successes, (kwargs.get("call_type"), getattr(response_obj, "id", None)))
def test_sync_completion_with_tools_while_reasoning_keeps_converse_and_logs_one_success(
monkeypatch: pytest.MonkeyPatch,
) -> None:
recorder: Final = _SuccessRecorder()
monkeypatch.setattr(litellm, "callbacks", [recorder])
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire:
response: Final = litellm.completion(
model=f"bedrock/{_MODEL_ID}",
messages=[{"role": "user", "content": f"synthetic native request marker-{marker}"}],
tools=[_TOOL],
reasoning_effort="high",
api_key="synthetic-bedrock-bearer",
aws_region_name="us-east-1",
aws_bedrock_runtime_endpoint=wire.url,
num_retries=0,
)
sent: Final = [(request.method, target_of(request)) for request in wire.drain()]
assert sent == [("POST", f"/model/{_MODEL_ID}/converse")], sent
assert isinstance(response, ModelResponse), type(response)
choice: Final = response.choices[0]
assert isinstance(choice, Choices), choice
assert choice.message.content == answer(marker), choice
logged: Final = eventually(
lambda: recorder.successes,
lambda successes: any(logged_id == response.id for _, logged_id in successes),
)
assert logged == (("completion", response.id),), logged

31
tests/unit/completion_extras/test_responses_bridge_provider_propagation.py
View file

@ -13,6 +13,7 @@ from litellm.completion_extras.litellm_responses_transformation.handler import (
ResponsesToCompletionBridgeHandler,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.utils import ModelResponse
MODEL = "openai.gpt-5.5"
@ -193,3 +194,33 @@ async def test_acompletion_without_cache_field_is_served_from_cache(_bridged_cac
assert len(_bridged_cache_edge.calls) == 1
assert [first.id, second.id] == ["resp_1", "resp_1"]
@pytest.mark.parametrize(
"bridged_request",
[
{"model": "openai/gpt-5.5-pro"},
{"model": "azure/gpt-5-pro", "api_version": "2025-04-01-preview"},
{**_GPT_6_REQUEST, "reasoning_effort": "low"},
],
ids=["openai_responses_mode", "azure_responses_mode", "azure_tools_with_reasoning"],
)
async def test_async_bridged_call_is_sent_through_the_callers_client(bridged_request):
sent: Final[list[httpx.Request]] = []
async def handle(request: httpx.Request) -> httpx.Response:
sent.append(request)
return httpx.Response(200, json=_responses_api_body(1))
response = await litellm.acompletion(
**{
"messages": [{"role": "user", "content": "hi"}],
"api_key": "x",
**bridged_request,
"api_base": "http://127.0.0.1:9",
},
client=AsyncHTTPHandler(transport=httpx.MockTransport(handle)),
)
assert [(request.url.host, request.url.path.endswith("/responses")) for request in sent] == [("127.0.0.1", True)]
assert response.choices[0].message.content == "hi 1"

515
tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py
View file

@ -2,6 +2,8 @@
import json
import re
from collections.abc import Iterable, Mapping
from typing import Final
import httpx
import pytest
@ -24,7 +26,7 @@ from litellm.llms.bedrock.common_utils import (
bedrock_runtime_chat_completions_is_default,
get_bedrock_chat_config,
)
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
APPLICATION_INFERENCE_PROFILE_ARN = "arn:aws:bedrock:us-west-2:123412341234:application-inference-profile/a1b2c3"
@ -260,6 +262,16 @@ def _recording_client(**response_kwargs):
return requests, HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle)))
def _async_recording_client(**response_kwargs):
requests: list[httpx.Request] = []
async def handle(request):
requests.append(request)
return httpx.Response(200, **response_kwargs)
return requests, AsyncHTTPHandler(transport=httpx.MockTransport(handle))
@pytest.mark.parametrize(
"model, model_path",
[
@ -549,20 +561,79 @@ def test_bearer_api_key_is_sent_as_the_authorization_header(monkeypatch):
assert headers["Authorization"] == "Bearer bedrock-api-key"
FILE_SEARCH_TOOL = {"type": "file_search", "vector_store_ids": ["vs_other_team"]}
def _weather_tool_named(name):
return {**GET_WEATHER_TOOL, "function": {**GET_WEATHER_TOOL["function"], "name": name}}
@pytest.mark.parametrize(
"request_params, expected_route",
[
({"tools": [GET_WEATHER_TOOL]}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": None}, "converse"),
({"tools": [GET_WEATHER_TOOL]}, "responses"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}, "responses"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": None}, "responses"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "none"}, "chat_completions"),
(
{
"tools": [GET_WEATHER_TOOL],
"reasoning_effort": "low",
"guardrailConfig": {"guardrailIdentifier": "gr-1"},
},
"converse",
),
({"tools": [GET_WEATHER_TOOL, FILE_SEARCH_TOOL], "reasoning_effort": "low"}, "converse"),
({"tools": GET_WEATHER_TOOL, "reasoning_effort": "low"}, "converse"),
({"tools": ["get_weather"], "reasoning_effort": "low"}, "converse"),
({"tools": [{"type": "function"}], "reasoning_effort": "low"}, "converse"),
({"tools": [_weather_tool_named("get weather!")], "reasoning_effort": "low"}, "converse"),
({"tools": [_weather_tool_named("weather.get")], "reasoning_effort": "low"}, "converse"),
({"tools": [_weather_tool_named("w" * 65)], "reasoning_effort": "low"}, "converse"),
({"tools": [_weather_tool_named("w" * 64)], "reasoning_effort": "low"}, "responses"),
({"tools": [_weather_tool_named("get-weather_2")], "reasoning_effort": "low"}, "responses"),
(
{"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"tools": [FILE_SEARCH_TOOL]}},
"converse",
),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"store": False}}, "responses"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"top_k": 5}}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": "store=false"}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "seed": 7}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "logit_bias": {"50256": -100}}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "web_search_options": {}}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "n": 2}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "n": 1}, "responses"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "presence_penalty": 0.5}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "presence_penalty": 0}, "responses"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "logprobs": True}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "logprobs": False}, "responses"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "prompt_cache_key": "k"}, "responses"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": {"effort": "low", "summary": "auto"}}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "store": "false"}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "store": 0}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "foo_bar": 1}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "foo_bar": None}, "responses"),
(
{
"tools": [GET_WEATHER_TOOL],
"reasoning_effort": "low",
"model": "global.openai.gpt-6-sol",
"messages": [{"role": "user", "content": "hi"}],
"drop_params": True,
"aws_region_name": "us-east-1",
"metadata": {"team": "a"},
"reasoning_summary": "auto",
},
"responses",
),
({"reasoning_effort": "low"}, "chat_completions"),
({"tools": None, "reasoning_effort": "low"}, "chat_completions"),
({"tools": [], "reasoning_effort": "low"}, "chat_completions"),
({}, "chat_completions"),
],
)
def test_gpt56_tools_need_reasoning_none_on_chat_completions(local_cost_map, request_params, expected_route):
def test_gpt56_tools_with_reasoning_bridge_to_responses(local_cost_map, request_params, expected_route):
assert (
BedrockModelInfo.get_bedrock_route("chat_completions/global.openai.gpt-5.6-sol", request_params)
== expected_route
@ -603,6 +674,21 @@ def test_gpt56_legacy_functions_route_like_tools(local_cost_map, request_params,
)
def test_gpt56_tools_with_reasoning_stay_on_converse_without_a_responses_endpoint(local_cost_map, monkeypatch):
model = "global.openai.gpt-6-sol"
monkeypatch.setitem(
litellm.model_cost, model, {**litellm.model_cost[model], "supported_endpoints": ["/v1/chat/completions"]}
)
params = {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}
assert BedrockModelInfo.get_bedrock_route(model, params) == "converse"
assert BedrockModelInfo.get_bedrock_route(model, {**params, "reasoning_effort": "none"}) == "chat_completions"
def test_gpt56_region_path_tools_with_reasoning_stay_on_converse(local_cost_map):
params = {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}
assert BedrockModelInfo.get_bedrock_route("bedrock/us-west-2/global.openai.gpt-6-sol", params) == "converse"
def test_thinking_block_goes_to_converse(local_cost_map):
thinking = {"type": "enabled", "budget_tokens": 1024}
assert BedrockModelInfo.get_bedrock_route("chat_completions/us.xai.grok-4.6", {"thinking": thinking}) == "converse"
@ -1055,19 +1141,407 @@ def test_gpt_oss_completion_hits_chat_completions_and_splits_reasoning(local_cos
assert response.choices[0].message.content == "Hi"
def test_gpt56_tools_with_reasoning_effort_go_to_converse(local_cost_map, fake_aws_env):
RESPONSES_TOOL_CALL_JSON = {
"id": "resp_test",
"object": "response",
"created_at": 1733529600,
"status": "completed",
"output": [
{
"type": "function_call",
"id": "fc_0",
"call_id": "call_0",
"name": "get_weather",
"arguments": '{"city": "Paris"}',
"status": "completed",
}
],
"usage": {"input_tokens": 5, "output_tokens": 3, "total_tokens": 8},
}
@pytest.mark.parametrize(
"model", ["bedrock/chat_completions/global.openai.gpt-5.6-sol", "bedrock/global.openai.gpt-6-sol"]
)
async def test_gpt56_and_newer_tools_with_reasoning_effort_bridge_to_runtime_responses(
local_cost_map, fake_aws_env, model
):
requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": model})
response = await litellm.acompletion(
model=model,
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
temperature=0.2,
drop_params=True,
client=client,
)
assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/responses"
body = json.loads(requests[0].content)
assert body["model"] == model.removeprefix("bedrock/").removeprefix("chat_completions/")
assert body["reasoning"] == {"effort": "low"}
assert [tool["name"] for tool in body["tools"]] == ["get_weather"]
assert "temperature" not in body
assert body["store"] is False
assert body["include"] == ["reasoning.encrypted_content"]
assert response.object == "chat.completion"
assert response.choices[0].finish_reason == "tool_calls"
assert response.choices[0].message.tool_calls[0].function.name == "get_weather"
assert response._hidden_params["response_cost"] > 0
async def test_a_provider_specific_param_keeps_converse_and_reaches_the_model(local_cost_map, fake_aws_env):
requests, client = _async_recording_client(json=CONVERSE_JSON)
await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
foo_bar=1,
client=client,
)
assert [request.url.raw_path for request in requests] == [b"/model/global.openai.gpt-6-sol/converse"]
assert json.loads(requests[0].content)["additionalModelRequestFields"]["foo_bar"] == 1
async def test_tools_with_reasoning_bridge_stores_only_when_the_caller_asks(local_cost_map, fake_aws_env):
requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"})
await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
store=True,
client=client,
)
body = json.loads(requests[0].content)
assert body["store"] is True
assert "include" not in body
async def test_tools_with_reasoning_and_a_param_the_bridge_drops_fail_like_converse_without_drop_params(
local_cost_map, fake_aws_env
):
requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"})
with pytest.raises(litellm.UnsupportedParamsError, match="seed"):
await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
seed=7,
client=client,
)
assert requests == []
@pytest.mark.parametrize(
"runtime_endpoint_param, runtime_endpoint_env, expected_host",
[
("https://vpce-param.example.com", None, "vpce-param.example.com"),
(None, "https://vpce-env.example.com", "vpce-env.example.com"),
(None, None, "proxy.example.com"),
],
)
async def test_tools_with_reasoning_bridge_goes_to_the_host_converse_would(
local_cost_map, fake_aws_env, monkeypatch, runtime_endpoint_param, runtime_endpoint_env, expected_host
):
if runtime_endpoint_env:
monkeypatch.setenv("AWS_BEDROCK_RUNTIME_ENDPOINT", runtime_endpoint_env)
params = {
"model": "bedrock/global.openai.gpt-6-sol",
"messages": [{"role": "user", "content": "weather in Paris"}],
"reasoning_effort": "low",
"api_base": "https://proxy.example.com",
"aws_bedrock_runtime_endpoint": runtime_endpoint_param,
}
converse_requests, converse_client = _async_recording_client(json=CONVERSE_JSON)
await litellm.acompletion(**params, tools=[_weather_tool_named("get weather!")], client=converse_client)
bridge_requests, bridge_client = _async_recording_client(
json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}
)
await litellm.acompletion(**params, tools=[GET_WEATHER_TOOL], client=bridge_client)
assert converse_requests[0].url.raw_path.endswith(b"/converse")
assert bridge_requests[0].url.path == "/openai/v1/responses"
assert bridge_requests[0].url.host == converse_requests[0].url.host == expected_host
async def test_tools_with_reasoning_and_a_name_openai_rejects_keep_converse_and_its_name_mapping(
local_cost_map, fake_aws_env
):
requests, client = _async_recording_client(json=CONVERSE_JSON)
await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[_weather_tool_named("get weather!")],
reasoning_effort="low",
client=client,
)
assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse")
assert json.loads(requests[0].content)["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather_"
async def test_a_caller_param_named_like_the_bridge_flag_neither_crashes_nor_steers_the_route(
local_cost_map, fake_aws_env
):
requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"})
response = await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
_litellm_responses_api_bridge_allowed=False,
client=client,
)
assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/responses"
assert response.choices[0].message.tool_calls[0].function.name == "get_weather"
async def test_tools_with_reasoning_bridge_keeps_the_reasoning_summary_alias(local_cost_map, fake_aws_env):
requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"})
await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
extra_body={"reasoningSummary": "auto"},
client=client,
)
assert json.loads(requests[0].content)["reasoning"] == {"effort": "low", "summary": "auto"}
def test_sync_tools_with_reasoning_stay_on_converse(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=CONVERSE_JSON)
response = litellm.completion(
model="bedrock/chat_completions/global.openai.gpt-5.6-sol",
messages=[{"role": "user", "content": "hello"}],
litellm.completion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
client=client,
)
assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-5.6-sol/converse")
assert json.loads(requests[0].content)["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather"
assert response.choices[0].message.content == "ok"
assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse")
async def test_tools_with_an_object_reasoning_effort_stay_on_converse_like_main(local_cost_map, fake_aws_env):
requests, client = _async_recording_client(json=CONVERSE_JSON)
await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort={"effort": "low", "summary": "auto"},
client=client,
)
assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse")
def _responses_reply_with_reasoning(encrypted_content: str, model: str) -> dict:
return {
**RESPONSES_TOOL_CALL_JSON,
"model": model,
"output": [
{"type": "reasoning", "id": "rs_0", "encrypted_content": encrypted_content, "summary": []},
*RESPONSES_TOOL_CALL_JSON["output"],
],
}
def _responses_stream_with_reasoning(encrypted_content: str, model: str) -> bytes:
completed: Final = {"type": "response.completed", "response": _responses_reply_with_reasoning(encrypted_content, model)}
return f"event: response.completed\ndata: {json.dumps(completed)}\n\n".encode()
async def _bedrock_turn(
history: list, model: str = "global.openai.gpt-6-sol", **response_kwargs
) -> tuple[list[httpx.Request], object]:
requests, client = _async_recording_client(**response_kwargs)
response = await litellm.acompletion(
model=f"bedrock/{model}",
messages=history,
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
stream="text/event-stream" in response_kwargs.get("headers", {}).values(),
client=client,
)
return requests, response
async def _openai_turn(history: list, reply: dict) -> tuple[list[httpx.Request], object]:
requests, client = _async_recording_client(json=reply)
response = await litellm.acompletion(
model="openai/gpt-5.5-pro",
api_key="x",
api_base="http://127.0.0.1:9",
messages=history,
tools=[GET_WEATHER_TOOL],
client=client,
)
return requests, response
def _history_after(reasoning_items: Iterable[Mapping[str, object]]) -> list[dict[str, object]]:
return [
{"role": "user", "content": "weather in Paris"},
{
"role": "assistant",
"content": None,
"reasoning_items": [dict(item) for item in reasoning_items],
"tool_calls": [
{
"id": "call_0",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city": "Paris"}'},
}
],
},
{"role": "tool", "tool_call_id": "call_0", "content": "14C, light rain"},
]
def _replayed_reasoning(request: httpx.Request) -> list:
return [
item.get("encrypted_content") for item in json.loads(request.content)["input"] if item.get("type") == "reasoning"
]
async def _streamed_reasoning_items(response) -> list:
deltas: Final = [chunk.choices[0].delta async for chunk in response if chunk.choices]
return [item for delta in deltas for item in getattr(delta, "reasoning_items", None) or ()]
@pytest.mark.parametrize("stream", [False, True], ids=["plain", "streamed"])
async def test_reasoning_bedrock_minted_replays_to_bedrock_but_never_to_openai(local_cost_map, fake_aws_env, stream):
reply_kwargs: Final = (
{
"content": _responses_stream_with_reasoning("bedrock-minted", "global.openai.gpt-6-sol"),
"headers": {"content-type": "text/event-stream"},
}
if stream
else {"json": _responses_reply_with_reasoning("bedrock-minted", "global.openai.gpt-6-sol")}
)
_, first = await _bedrock_turn([{"role": "user", "content": "weather in Paris"}], **reply_kwargs)
reasoning_items: Final = (
await _streamed_reasoning_items(first) if stream else list(first.choices[0].message.reasoning_items)
)
history: Final = _history_after(reasoning_items)
to_bedrock, _ = await _bedrock_turn(history, json=RESPONSES_TOOL_CALL_JSON)
to_openai, _ = await _openai_turn(history, {**RESPONSES_TOOL_CALL_JSON, "model": "gpt-5.5-pro"})
assert len(reasoning_items) == 1
assert _replayed_reasoning(to_bedrock[0]) == ["bedrock-minted"]
assert _replayed_reasoning(to_openai[0]) == []
async def test_reasoning_one_bedrock_model_minted_never_replays_to_another(local_cost_map, fake_aws_env):
_, first = await _bedrock_turn(
[{"role": "user", "content": "weather in Paris"}],
json=_responses_reply_with_reasoning("gpt-6-minted", "global.openai.gpt-6-sol"),
)
history: Final = _history_after(first.choices[0].message.reasoning_items)
to_same_model, _ = await _bedrock_turn(history, json=RESPONSES_TOOL_CALL_JSON)
to_same_model_us_profile, _ = await _bedrock_turn(
history, model="us.openai.gpt-6-sol", json=RESPONSES_TOOL_CALL_JSON
)
to_other_model, _ = await _bedrock_turn(history, model="us.openai.gpt-5.6-sol", json=RESPONSES_TOOL_CALL_JSON)
assert _replayed_reasoning(to_same_model[0]) == ["gpt-6-minted"]
assert _replayed_reasoning(to_same_model_us_profile[0]) == ["gpt-6-minted"]
assert _replayed_reasoning(to_other_model[0]) == []
async def test_reasoning_openai_minted_replays_to_openai_but_never_to_bedrock(local_cost_map, fake_aws_env):
_, first = await _openai_turn(
[{"role": "user", "content": "weather in Paris"}], _responses_reply_with_reasoning("openai-minted", "gpt-5.5-pro")
)
history: Final = _history_after(first.choices[0].message.reasoning_items)
to_openai, _ = await _openai_turn(history, {**RESPONSES_TOOL_CALL_JSON, "model": "gpt-5.5-pro"})
to_bedrock, _ = await _bedrock_turn(history, json=RESPONSES_TOOL_CALL_JSON)
assert _replayed_reasoning(to_openai[0]) == ["openai-minted"]
assert _replayed_reasoning(to_bedrock[0]) == []
async def test_async_tools_with_reasoning_bridge_uses_the_supplied_client(local_cost_map, fake_aws_env):
requests: Final[list[httpx.Request]] = []
async def handle(request: httpx.Request) -> httpx.Response:
requests.append(request)
return httpx.Response(200, json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"})
response = await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
client=AsyncHTTPHandler(transport=httpx.MockTransport(handle)),
)
assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/responses"
assert response.choices[0].message.tool_calls[0].function.name == "get_weather"
class _FrozenCredentials:
access_key = "AKIDLEGACYCLIENT"
secret_key = "legacy-secret"
token = None
class _LegacyCredentials:
def get_frozen_credentials(self):
return _FrozenCredentials()
class _LegacyBedrockClient:
class meta:
region_name = "eu-west-1"
def _get_credentials(self):
return _LegacyCredentials()
async def test_tools_with_reasoning_and_a_legacy_aws_bedrock_client_stay_on_converse(local_cost_map, fake_aws_env):
requests, client = _async_recording_client(json=CONVERSE_JSON)
await litellm.acompletion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
aws_bedrock_client=_LegacyBedrockClient(),
client=client,
)
assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse")
assert requests[0].url.host == "bedrock-runtime.eu-west-1.amazonaws.com"
assert "Credential=AKIDLEGACYCLIENT/" in requests[0].headers["Authorization"]
def test_responses_forced_onto_chat_completions_serves_tools_with_reasoning_on_converse(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=CONVERSE_JSON)
litellm.responses(
model="bedrock/global.openai.gpt-6-sol",
input="weather in Paris",
tools=[{"type": "function", "name": "get_weather", "parameters": GET_WEATHER_TOOL["function"]["parameters"]}],
reasoning={"effort": "low"},
use_chat_completions_api=True,
client=client,
)
assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse")
body = json.loads(requests[0].content)
assert body["additionalModelRequestFields"] == {"reasoning": {"effort": "low"}}
assert body["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather"
def test_gpt56_tools_with_reasoning_none_stay_on_chat_completions(local_cost_map, fake_aws_env):
@ -1113,23 +1587,6 @@ def test_gpt_56_and_newer_completion_without_the_prefix_posts_runtime_chat_compl
assert response._hidden_params["response_cost"] > 0
def test_gpt6_without_the_prefix_tools_with_reasoning_effort_go_to_converse(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=CONVERSE_JSON)
response = litellm.completion(
model="bedrock/global.openai.gpt-6-sol",
messages=[{"role": "user", "content": "hello"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
client=client,
)
assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse")
body = json.loads(requests[0].content)
assert body["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather"
assert body["additionalModelRequestFields"]["reasoning"] == {"effort": "low"}
assert response.choices[0].message.content == "ok"
def test_gpt6_without_the_prefix_guardrail_config_goes_to_converse(local_cost_map, fake_aws_env):
guardrail = {"guardrailIdentifier": "gr-1", "guardrailVersion": "1"}
requests, client = _recording_client(json=CONVERSE_JSON)

13
tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py
View file

@ -57,6 +57,19 @@ class TestCompleteURL:
)
assert url == "https://vpce.example.com/openai/v1/responses"
@pytest.mark.parametrize(
"runtime_endpoint_param, runtime_endpoint_env",
[("https://vpce.example.com", None), (None, "https://env-vpce.example.com")],
)
def test_api_base_wins_over_a_runtime_endpoint(self, monkeypatch, runtime_endpoint_param, runtime_endpoint_env):
if runtime_endpoint_env:
monkeypatch.setenv("AWS_BEDROCK_RUNTIME_ENDPOINT", runtime_endpoint_env)
url = _cfg().get_complete_url(
"https://proxy.example.com",
{"aws_region_name": "us-east-1", "aws_bedrock_runtime_endpoint": runtime_endpoint_param},
)
assert url == "https://proxy.example.com/openai/v1/responses"
class TestAuth:
def test_bearer_token_is_used_when_present(self):