diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 3f7017eccc2..3bc1f713a42 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -11,7 +11,7 @@ from typing import ( cast, # noqa: TID251 # optional_params reaches the bridge as an untyped dict ) -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict from litellm.types.llms.openai import ResponsesAPIResponse @@ -29,6 +29,7 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict): logging_obj: "LiteLLMLoggingObj" custom_llm_provider: str encoding: object + client: ReadOnly[object] def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str: @@ -158,6 +159,7 @@ class ResponsesToCompletionBridgeHandler: logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, encoding=typed_kwargs.get("encoding"), + client=typed_kwargs.get("client"), ) def completion( @@ -192,7 +194,7 @@ class ResponsesToCompletionBridgeHandler: litellm_params=litellm_params, headers=headers, litellm_logging_obj=logging_obj, - client=kwargs.get("client"), + client=validated_kwargs["client"], ) # Set on request_data rather than passed as explicit kwargs: the spread of @@ -251,7 +253,7 @@ class ResponsesToCompletionBridgeHandler: else: if self._is_preformatted_cached_chat_stream(result): return self._apply_post_stream_processing(result, model, custom_llm_provider) - from .transformation import is_structured_output_request + from .transformation import is_structured_output_request, reasoning_origin_for completion_stream: Final = self.transformation_handler.get_model_response_iterator( streaming_response=result, @@ -260,6 +262,7 @@ class ResponsesToCompletionBridgeHandler: structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), + reasoning_origin=reasoning_origin_for(custom_llm_provider, model), ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, @@ -293,6 +296,7 @@ class ResponsesToCompletionBridgeHandler: litellm_params=litellm_params, headers=headers, litellm_logging_obj=logging_obj, + client=validated_kwargs["client"], ) except Exception as e: raise e @@ -354,7 +358,7 @@ class ResponsesToCompletionBridgeHandler: else: if self._is_preformatted_cached_chat_stream(result): return self._apply_post_stream_processing(result, model, custom_llm_provider) - from .transformation import is_structured_output_request + from .transformation import is_structured_output_request, reasoning_origin_for completion_stream: Final = self.transformation_handler.get_model_response_iterator( streaming_response=result, @@ -363,6 +367,7 @@ class ResponsesToCompletionBridgeHandler: structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), + reasoning_origin=reasoning_origin_for(custom_llm_provider, model), ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 8d23526b41a..3dd5718e867 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -5,6 +5,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req import json import os from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass from itertools import accumulate, chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, Union, cast @@ -305,6 +306,59 @@ def _reasoning_items_from_output_items(output_items: Sequence[object]) -> tuple[ ) +_REASONING_ORIGIN_PREFIX: Final = "litellm_reasoning_origin:" + + +@dataclass(frozen=True, slots=True) +class ReasoningReplayTarget: + origin: str | None + + +def reasoning_origin_for(provider: str | None, model: str) -> str | None: + if provider != "bedrock": + return None + from litellm.llms.bedrock.common_utils import get_bedrock_base_model + + return f"{_REASONING_ORIGIN_PREFIX}bedrock/{get_bedrock_base_model(model)}:" + + +def _with_reasoning_origin(output_items: Sequence[object], origin: str | None) -> Sequence[object]: + if origin is None: + return output_items + return tuple(_reasoning_output_item_with_origin(item, origin) for item in output_items) + + +def _reasoning_output_item_with_origin(item: object, origin: str) -> object: + from openai.types.responses import ResponseReasoningItem + + if isinstance(item, ResponseReasoningItem): + return ( + item.model_copy(update={"encrypted_content": f"{origin}{item.encrypted_content}"}) + if item.encrypted_content + else item + ) + if not isinstance(item, dict): + return item + fields: Final = cast(dict[str, object], item) # cast-ok: provider JSON item, keys forwarded verbatim + encrypted_content: Final = fields.get("encrypted_content") + if fields.get("type") != "reasoning" or not isinstance(encrypted_content, str) or not encrypted_content: + return fields + return {**fields, "encrypted_content": f"{origin}{encrypted_content}"} + + +def _replayable_reasoning_item( + item: Mapping[str, object], target: ReasoningReplayTarget +) -> Mapping[str, object] | None: + encrypted_content: Final = item.get("encrypted_content") + if not isinstance(encrypted_content, str) or not encrypted_content: + return item + if target.origin is None: + return None if encrypted_content.startswith(_REASONING_ORIGIN_PREFIX) else item + if not encrypted_content.startswith(target.origin): + return None + return {**item, "encrypted_content": encrypted_content.removeprefix(target.origin)} + + _TOOL_SEARCH_ITEM_TYPES: Final = frozenset({"tool_search_call", "tool_search_output"}) _TOOL_SEARCH_REPLAY_ITEM_TYPES: Final = frozenset({"reasoning", *_TOOL_SEARCH_ITEM_TYPES}) _TOOL_SEARCH_ITEMS_KEY: Final = "tool_search_items" @@ -428,11 +482,18 @@ def _reasoning_ids(items: Sequence[Mapping[str, object]]) -> frozenset[str]: return frozenset(item_id for item_id in ids if isinstance(item_id, str)) -def _assistant_replay_items(msg: "AllMessageValues", replay_tool_search: bool) -> tuple[Mapping[str, object], ...]: +def _assistant_replay_items( + msg: "AllMessageValues", replay_tool_search: bool, replay_to: ReasoningReplayTarget | None +) -> tuple[Mapping[str, object], ...]: searched: Final = _tool_search_input_items(msg) if replay_tool_search else () searched_reasoning_ids: Final = _reasoning_ids(searched) reasoning: Final = (item for item in _reasoning_input_items(msg) if item.get("id") not in searched_reasoning_ids) - return (*searched, *reasoning) + replayable: Final = ( + reasoning + if replay_to is None + else (kept for item in reasoning if (kept := _replayable_reasoning_item(item, replay_to)) is not None) + ) + return (*searched, *replayable) def _as_chat_reasoning_items( @@ -790,12 +851,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): keep_prompt_cache_breakpoints: bool = False, replay_tool_search: bool = False, include_namespace: bool = False, + replay_to: ReasoningReplayTarget | None = None, ) -> tuple[list[object], str | None]: converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input( messages, drop_params=drop_params, replay_tool_search=replay_tool_search, include_namespace=include_namespace, + replay_to=replay_to, ) return ( converted_input_items @@ -811,6 +874,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): drop_params: bool = False, replay_tool_search: bool = False, include_namespace: bool = False, + replay_to: ReasoningReplayTarget | None = None, ) -> tuple[list[object], str | None]: input_items: Final[list[object]] = [] instructions: str | None = None @@ -892,7 +956,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) ) elif role == "assistant" and tool_calls and isinstance(tool_calls, list): - input_items.extend(_assistant_replay_items(msg, replay_tool_search)) + input_items.extend(_assistant_replay_items(msg, replay_tool_search, replay_to)) if content: input_items.append( { @@ -933,7 +997,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): raise ValueError(f"tool call not supported: {tool_call}") elif content is not None: if role == "assistant": - input_items.extend(_assistant_replay_items(msg, replay_tool_search)) + input_items.extend(_assistant_replay_items(msg, replay_tool_search, replay_to)) input_items.append( { "type": "message", @@ -944,7 +1008,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): } ) elif role == "assistant": - input_items.extend(_assistant_replay_items(msg, replay_tool_search)) + input_items.extend(_assistant_replay_items(msg, replay_tool_search, replay_to)) return input_items, instructions @@ -1065,6 +1129,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint, replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast include_namespace=True, + replay_to=ReasoningReplayTarget(origin=reasoning_origin_for(provider, model)), ) # OpenAI's Responses API rejects an empty input. For a system-only # request, carry the system message as a system-role input item instead @@ -1419,8 +1484,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) # Convert response output to choices using the static helper + tagged_output_items: Final = _with_reasoning_origin( + output_items, reasoning_origin_for(_served_provider(litellm_params), model) + ) choices: Final = self._convert_response_output_to_choices( - output_items=output_items, + output_items=tagged_output_items, handle_raw_dict_callback=self._handle_raw_dict_response_item, structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict @@ -1439,7 +1507,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): raw_response.incomplete_details.reason if raw_response.incomplete_details is not None else None ) if len(choices) == 0: - choices.append(self._build_empty_choice(output_items, incomplete_finish_reason)) + choices.append(self._build_empty_choice(tagged_output_items, incomplete_finish_reason)) else: for choice in choices: choice.finish_reason = incomplete_finish_reason @@ -1488,9 +1556,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): sync_stream: bool, json_mode: bool | None = False, structured_output: bool = False, + reasoning_origin: str | None = None, ) -> BaseModelResponseIterator: return OpenAiResponsesToChatCompletionStreamIterator( - streaming_response, sync_stream, json_mode, structured_output=structured_output + streaming_response, + sync_stream, + json_mode, + structured_output=structured_output, + reasoning_origin=reasoning_origin, ) def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]: @@ -1869,8 +1942,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): sync_stream: bool, json_mode: bool | None = False, structured_output: bool = False, + reasoning_origin: str | None = None, ): super().__init__(streaming_response, sync_stream, json_mode) + self._reasoning_origin: Final = reasoning_origin self._chat_completion_id: str | None = None self._served_service_tier: str | None = None self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state @@ -1916,6 +1991,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): def translate_responses_chunk_to_openai_stream( parsed_chunk: dict | BaseModel, tool_call_index_map: dict[int, int] | None = None, # mutable-ok: per-stream state, remapped in place + reasoning_origin: str | None = None, ) -> "ModelResponseStream": """ Translate a Responses API streaming chunk to OpenAI chat completion streaming format. @@ -2131,7 +2207,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ) terminal_reasoning_items_typed: Final = _as_chat_reasoning_items( - _reasoning_items_from_output_items(output_items) + _reasoning_items_from_output_items(_with_reasoning_origin(output_items, reasoning_origin)) ) usage = None @@ -2195,7 +2271,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): self._empty_content_chunk() if self._is_text_delta_after_structured_answer(chunk) else OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream( - chunk, tool_call_index_map=self._tool_call_index_map + chunk, tool_call_index_map=self._tool_call_index_map, reasoning_origin=self._reasoning_origin ) ) if self._structured_output and _is_message_item_done(chunk): diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 17a8192ca81..f81eb505f13 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -11,7 +11,7 @@ import os import re import warnings from collections.abc import Iterator, Mapping, Sequence -from types import MappingProxyType +from types import MappingProxyType, NoneType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias from typing_extensions import ReadOnly, TypedDict @@ -27,6 +27,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError import litellm from litellm import verbose_logger +from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES, OPENAI_CHAT_COMPLETION_PARAMS from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.optional_imports import ensure_optional_import from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count @@ -38,6 +39,8 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.request_metadata import bedrock_request_metadata_is_owned from litellm.secret_managers.main import get_secret, get_secret_str from litellm.types.llms.bedrock import AWS_AUTH_PARAM_KEYS, AwsAuthParams +from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams +from litellm.types.utils import is_litellm_owned_kwarg if TYPE_CHECKING: from litellm.llms.custom_httpx.http_handler import HTTPHandler @@ -61,6 +64,7 @@ BedrockRoute = Literal[ "openai", "mantle", "chat_completions", + "responses", ] @@ -1092,7 +1096,8 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje written onto the Converse body, function tools (``tools`` or legacy ``functions``) on a model without ``supports_bedrock_runtime_chat_completions_tools_with_reasoning`` are rejected there unless - ``reasoning_effort`` is exactly ``"none"``, and a ``response_format`` goes native only as + ``reasoning_effort`` is exactly ``"none"`` (function-only ``tools`` on a model listing ``/v1/responses`` are + bridged to bedrock-runtime's native Responses API instead), and a ``response_format`` goes native only as ``{"type": "json_schema", "json_schema": ...}`` (a pydantic model is converted to that) on a model with ``supports_bedrock_runtime_chat_completions_response_format``: a schema on any other model is only honored by Converse, and every ``json_object`` form (``response_schema`` included) keeps Converse's @@ -1110,6 +1115,12 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje return True if _response_format_needs_converse(model, request_params.get("response_format")): return True + return _chat_completions_rejects_function_tools(model, request_params) and not _bridgeable_to_responses( + model, request_params + ) + + +def _chat_completions_rejects_function_tools(model: str, request_params: Mapping[str, object]) -> bool: if not (request_params.get("tools") or request_params.get("functions")): return False return ( @@ -1118,12 +1129,140 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje ) +class _NamedFunction(TypedDict): + name: ReadOnly[str] + + +class _FunctionTool(TypedDict): + type: ReadOnly[Literal["function"]] + function: ReadOnly[_NamedFunction] + + +_FUNCTION_TOOLS_ADAPTER: Final = TypeAdapter(tuple[_FunctionTool, ...]) +_OPENAI_FUNCTION_NAME: Final = re.compile(r"[a-zA-Z0-9_-]{1,64}") + + +def _function_tools_only(tools: object) -> bool: + if not isinstance(tools, (list, tuple)): + return False + try: + validated: Final = _FUNCTION_TOOLS_ADAPTER.validate_python(tools) + except ValidationError: + return False + return all(_OPENAI_FUNCTION_NAME.fullmatch(tool["function"]["name"]) for tool in validated) + + +_EXTRA_BODY_ADAPTER: Final = TypeAdapter(dict[str, object]) +_EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES: Final = frozenset( + {*ResponsesAPIOptionalRequestParams.__annotations__, "reasoning_effort", "reasoningSummary", "reasoning_summary"} + - {"tools"} +) +_CHAT_PARAMS_THE_RESPONSES_BRIDGE_DROPS: Final = MappingProxyType( + { + "n": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "logprobs": False, + "top_logprobs": None, + "logit_bias": None, + "seed": None, + "modalities": None, + "audio": None, + "prediction": None, + "web_search_options": None, + "function_call": None, + "include_server_side_tool_invocations": None, + } +) +_ENCRYPTED_REASONING: Final = "reasoning.encrypted_content" +_INCLUDE_ADAPTER: Final = TypeAdapter(tuple[object, ...]) + + +def _extra_body_bridgeable(extra_body: object) -> bool: + if extra_body is None: + return True + return isinstance(extra_body, dict) and ( + _EXTRA_BODY_ADAPTER.validate_python(extra_body).keys() <= _EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES + ) + + +def _sets_a_chat_param_the_responses_bridge_drops(request_params: Mapping[str, object]) -> bool: + return any( + request_params.get(param) not in (None, no_op_value) + for param, no_op_value in _CHAT_PARAMS_THE_RESPONSES_BRIDGE_DROPS.items() + ) + + +_CHAT_PARAM_NAMES: Final = frozenset( + {*DEFAULT_CHAT_COMPLETION_PARAM_VALUES, *OPENAI_CHAT_COMPLETION_PARAMS, "extra_body"} +) + + +def _provider_specific_param(name: str) -> bool: + return not ( + name in _CHAT_PARAM_NAMES + or name in _EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES + or name.startswith("aws_") + or is_litellm_owned_kwarg(name) + ) + + +def _passes_a_provider_specific_param(request_params: Mapping[str, object]) -> bool: + return any(value is not None and _provider_specific_param(name) for name, value in request_params.items()) + + +def _bridgeable_to_responses(model: str, request_params: Mapping[str, object]) -> bool: + return ( + not request_params.get("functions") + and request_params.get("aws_bedrock_client") is None + and _function_tools_only(request_params.get("tools")) + and _extra_body_bridgeable(request_params.get("extra_body")) + and not _passes_a_provider_specific_param(request_params) + and not _sets_a_chat_param_the_responses_bridge_drops(request_params) + and isinstance(request_params.get("reasoning_effort"), (str, NoneType)) + and isinstance(request_params.get("store"), (bool, NoneType)) + and _bedrock_serves_responses(model) + ) + + +def _bedrock_serves_responses(model: str) -> bool: + region, model_id = split_bedrock_region_path(model) + return region is None and any( + _price_map_entry_lists_endpoint(_model_cost_entry(key), "/v1/responses") + for key in (model_id, f"bedrock/{model_id}") + ) + + +def bedrock_responses_bridge_optional_params(optional_params: Mapping[str, object]) -> Mapping[str, object]: + if optional_params.get("store") is not None: + return optional_params + include: Final = optional_params.get("include") + requested: Final = _INCLUDE_ADAPTER.validate_python(include) if isinstance(include, (list, tuple)) else () + return { + **optional_params, + "store": False, + "include": [*requested, *(() if _ENCRYPTED_REASONING in requested else (_ENCRYPTED_REASONING,))], + } + + +def bedrock_responses_bridge_api_base(api_base: str | None, aws_bedrock_runtime_endpoint: object) -> str | None: + return ( + (aws_bedrock_runtime_endpoint if isinstance(aws_bedrock_runtime_endpoint, str) else None) + or get_secret_str("AWS_BEDROCK_RUNTIME_ENDPOINT") + or api_base + ) + + def _chat_completions_unless_converse_needed( model: str, request_params: Mapping[str, object] | None -) -> Literal["converse", "chat_completions"]: - if request_params is not None and bedrock_request_needs_converse(model, request_params): +) -> Literal["converse", "chat_completions", "responses"]: + if request_params is None: + return "chat_completions" + if bedrock_request_needs_converse(model, request_params): return "converse" - return "chat_completions" + if not _chat_completions_rejects_function_tools(model, request_params): + return "chat_completions" + return "converse" if request_params.get("_litellm_responses_api_bridge_allowed") is False else "responses" def bedrock_route_for_request( @@ -1607,8 +1746,9 @@ class BedrockModelInfo(BaseLLMModelInfo): Grok and GPT 5.6 and newer go to bedrock-runtime's native OpenAI Chat Completions by default (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in; ``request_params`` (the caller's chat params) sends such a request to Converse when it needs a - feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family - model stays on Converse without the prefix. + feature only Converse serves, to ``responses`` (the chat-to-Responses bridge onto bedrock-runtime's + native ``/openai/v1/responses``) when it carries function tools with reasoning on, and ``converse/`` + pins a model to Converse. Every other OpenAI-family model stays on Converse without the prefix. """ route_mappings: dict[ str, diff --git a/litellm/main.py b/litellm/main.py index 3eae64d828e..5c8f69fd328 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -119,6 +119,8 @@ from litellm.llms.base_llm.base_model_iterator import ( from litellm.llms.base_llm.chat.transformation import with_attribution_headers from litellm.llms.bedrock.common_utils import ( BedrockModelInfo, + bedrock_responses_bridge_api_base, + bedrock_responses_bridge_optional_params, bedrock_route_for_request, without_bedrock_route_prefix, ) @@ -5689,6 +5691,7 @@ def completion( "allowed_openai_params": allowed_openai_params, "base_model": base_model, } + bedrock_responses_bridge_allowed: Final = acompletion is True and not skip_responses_api_bridge provider_reasoning_effort: Final = reasoning_effort_with_summary_alias( reasoning_effort, non_default_params, @@ -5699,6 +5702,7 @@ def completion( **{**optional_param_args, "reasoning_effort": provider_reasoning_effort}, model_info=_MODEL_INFO_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else None, **non_default_params, + _litellm_responses_api_bridge_allowed=bedrock_responses_bridge_allowed, ) processed_non_default_params: Final = pre_process_non_default_params( model=model, @@ -5863,14 +5867,39 @@ def completion( api_base=api_base, ) + request_params: Final = MappingProxyType( + _OPTIONAL_PARAMS_ADAPTER.validate_python( + { + **optional_param_args, + **non_default_params, + "_litellm_responses_api_bridge_allowed": bedrock_responses_bridge_allowed, + } + ) + ) + bedrock_bridges_to_responses: Final = ( + custom_llm_provider == "bedrock" + and bedrock_route_for_request( + model, + request_params, + _ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(kwargs.get("additional_drop_params") or ()), + ) + == "responses" + ) + # Use base_model (the true underlying model) for Azure model-type # detection when the deployment name differs from the model name. _azure_detection_model: Final = base_model or model - if responses_api_model_info.get("mode") == "responses" and not skip_responses_api_bridge: + if ( + responses_api_model_info.get("mode") == "responses" or bedrock_bridges_to_responses + ) and not skip_responses_api_bridge: from litellm.completion_extras import responses_api_bridge - optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params) + optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params( + {**bedrock_responses_bridge_optional_params(optional_params)} + if bedrock_bridges_to_responses + else optional_params + ) if isinstance(reasoning_effort, dict) and "summary" in reasoning_effort: optional_params["reasoning_effort"] = reasoning_effort @@ -5896,7 +5925,16 @@ def completion( acompletion=acompletion, logging_obj=logging, optional_params=optional_params, - litellm_params=litellm_params, + litellm_params=( + { + **litellm_params, + "api_base": bedrock_responses_bridge_api_base( + api_base, litellm_params.get("aws_bedrock_runtime_endpoint") + ), + } + if bedrock_bridges_to_responses + else litellm_params + ), timeout=timeout, client=client, # pass AsyncOpenAI, OpenAI client custom_llm_provider=custom_llm_provider, @@ -5935,9 +5973,7 @@ def completion( optional_params=optional_params, organization=organization, provider_config=provider_config, - request_params=MappingProxyType( - _OPTIONAL_PARAMS_ADAPTER.validate_python({**optional_param_args, **non_default_params}) - ), + request_params=request_params, shared_session=shared_session, stream=stream, temperature=temperature, diff --git a/litellm/utils.py b/litellm/utils.py index fc3acd3065d..1526b6da8d2 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4576,6 +4576,7 @@ def get_optional_params( prompt_cache_key: str | None = None, base_model: str | None = None, model_info: Mapping[str, object] | None = None, + _litellm_responses_api_bridge_allowed: bool = True, **kwargs: object, ): drop_params = normalize_drop_params(drop_params) # rebind-ok: config and DB deployments pass "true" as a string @@ -4587,6 +4588,7 @@ def get_optional_params( # OpenAI param. passed_params.pop("base_model", None) passed_params.pop("model_info", None) + passed_params.pop("_litellm_responses_api_bridge_allowed") provider_config: BaseConfig | None = None if custom_llm_provider is not None and custom_llm_provider in [provider.value for provider in LlmProviders]: provider_config = ProviderConfigManager.get_provider_chat_config( @@ -4638,7 +4640,11 @@ def get_optional_params( ) bedrock_route: Final = ( - _bedrock_route_for_request(model, passed_params, additional_drop_params) + _bedrock_route_for_request( + model, + {**passed_params, "_litellm_responses_api_bridge_allowed": _litellm_responses_api_bridge_allowed}, + additional_drop_params, + ) if custom_llm_provider == "bedrock" else None ) diff --git a/tests/integration/messages_endpoint/providers/bedrock/test_bedrock_messages_gpt_chat_completions_wire.py b/tests/integration/messages_endpoint/providers/bedrock/test_bedrock_messages_gpt_chat_completions_wire.py index 00945840808..e3fb141d950 100644 --- a/tests/integration/messages_endpoint/providers/bedrock/test_bedrock_messages_gpt_chat_completions_wire.py +++ b/tests/integration/messages_endpoint/providers/bedrock/test_bedrock_messages_gpt_chat_completions_wire.py @@ -4,7 +4,15 @@ from collections.abc import Mapping from typing import Final import anthropic -from integration._support.bedrock_runtime_peer import NATIVE_CHAT, answer, body_of, marker_of, respond, target_of +from integration._support.bedrock_runtime_peer import ( + NATIVE_CHAT, + NATIVE_RESPONSES, + answer, + body_of, + marker_of, + respond, + target_of, +) from integration._support.client import Gateway, Scenario, eventually from integration._support.database import read_rows from integration._support.wire import Request, Wire, wire_server @@ -14,6 +22,7 @@ BEDROCK_MODEL: Final = "us.openai.gpt-5.6-sol" TOKEN: Final = "synthetic-bedrock-bearer" NO_CACHE: Final[Mapping[str, JsonValue]] = {"cache": {"no-cache": True}} ANTHROPIC_VERSION: Final[Mapping[str, str]] = {"anthropic-version": "2023-06-01"} +TOOL_SCHEMA: Final[dict[str, object]] = {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]} def _question(marker: str) -> str: @@ -40,6 +49,41 @@ def _native_body(wire: Wire, marker: str) -> Mapping[str, JsonValue]: return body_of(received[0]) +def _bridged_body(wire: Wire, marker: str) -> Mapping[str, JsonValue]: + received: Final = _carrying(wire, marker) + assert [(request.method, target_of(request)) for request in received] == [("POST", NATIVE_RESPONSES)] + assert received[0].headers["authorization"] == f"Bearer {TOKEN}", received[0].headers + body: Final = body_of(received[0]) + assert (body["model"], body["reasoning"], body["max_output_tokens"]) == ( + BEDROCK_MODEL, + {"effort": "medium"}, + 4096, + ), body + assert (body["store"], body["include"]) == (False, ["reasoning.encrypted_content"]), body + assert [(tool["type"], tool["name"], tool["parameters"]) for tool in _tools_of(body)] == [ + ("function", "lookup_invoice", TOOL_SCHEMA) + ], body + return body + + +def _tools_of(body: Mapping[str, JsonValue]) -> list[dict[str, JsonValue]]: + tools: Final = body["tools"] + assert isinstance(tools, list), body + return [tool for tool in tools if isinstance(tool, dict)] + + +def _bridged_spend_row(model: str) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows( + 'SELECT status, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE model_group=%s', + (model,), + ), + lambda found: len(found) == 1, + seconds=70, + ) + return rows[0] + + def _native_request(marker: str, max_tokens: int, effort: str) -> Mapping[str, JsonValue]: return { "model": BEDROCK_MODEL, @@ -122,6 +166,50 @@ def test_anthropic_sdk_stream_with_thinking_budget_is_served_by_native_chat_comp assert _spend_rows(identity, 1) == [_success_row(identity, model, cache_hit="False")] +def test_anthropic_sdk_tools_with_thinking_are_bridged_to_native_responses(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + client: Final = anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0) + message: Final = client.messages.create( + model=model, + max_tokens=4096, + thinking={"type": "enabled", "budget_tokens": 2048}, + tools=[{"name": "lookup_invoice", "description": "Look up an invoice", "input_schema": TOOL_SCHEMA}], + messages=[{"role": "user", "content": _question(marker)}], + extra_body=NO_CACHE, + ) + assert "stream" not in _bridged_body(wire, marker) + assert [(block.type, getattr(block, "text", None)) for block in message.content] == [("text", answer(marker))] + assert (message.usage.input_tokens, message.usage.output_tokens) == (30, 5), message + assert _bridged_spend_row(model) == {"status": "success", "prompt_tokens": 30, "completion_tokens": 5} + + +def test_anthropic_sdk_stream_with_tools_and_thinking_is_bridged_to_native_responses(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + client: Final = anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0) + stream: Final = client.messages.create( + model=model, + max_tokens=4096, + thinking={"type": "enabled", "budget_tokens": 2048}, + tools=[{"name": "lookup_invoice", "description": "Look up an invoice", "input_schema": TOOL_SCHEMA}], + messages=[{"role": "user", "content": _question(marker)}], + extra_body=NO_CACHE, + stream=True, + ) + events: Final = list(stream) + assert _bridged_body(wire, marker)["stream"] is True + assert events[0].type == "message_start" and events[-1].type == "message_stop", events + assert "".join( + event.delta.text + for event in events + if event.type == "content_block_delta" and event.delta.type == "text_delta" + ) == answer(marker) + assert _bridged_spend_row(model) == {"status": "success", "prompt_tokens": 30, "completion_tokens": 5} + + def test_raw_thinking_summary_reaches_native_chat_completions_as_the_plain_effort(gateway: Gateway) -> None: marker: Final = uuid.uuid4().hex with wire_server(respond) as wire, gateway.scenario() as scenario: diff --git a/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py b/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py index c8f9e5c1f5e..7630b3f6b78 100644 --- a/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py +++ b/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py @@ -8,18 +8,21 @@ from urllib.parse import quote import httpx import openai import pytest -from integration._support.bedrock_runtime_peer import answer, respond, target_of -from integration._support.client import Gateway, Scenario, eventually +from integration._support import responses_vendor as rv +from integration._support.bedrock_runtime_peer import NATIVE_RESPONSES, answer, respond, target_of +from integration._support.client import Gateway, Scenario, eventually, list_value, object_value from integration._support.database import read_rows from integration._support.sigv4 import signature from integration._support.wire import Request, Wire, wire_server -from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam +from openai.types.chat import ChatCompletionChunk, ChatCompletionFunctionToolParam, ChatCompletionMessageParam from openai.types.chat.chat_completion_chunk import ChoiceDelta from pydantic import JsonValue, TypeAdapter GPT: Final = "us.openai.gpt-5.6-sol" GLOBAL_GPT: Final = "global.openai.gpt-5.6-sol" GPT_OSS: Final = "openai.gpt-oss-120b-1:0" +OTHER_GPT: Final = "global.openai.gpt-6.1-sol" +OPENAI_CODEX: Final = "gpt-5.3-codex" TOKEN: Final = "synthetic-bedrock-bearer" ACCESS_KEY: Final = "AKIASYNTHETICKEY0001" SECRET_KEY: Final = "synthetic-secret-key-for-testing" @@ -45,6 +48,15 @@ TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType( }, } ) +SDK_TOOL: Final[ChatCompletionFunctionToolParam] = { + "type": "function", + "function": { + "name": "lookup_invoice", + "description": "Look up an invoice", + "parameters": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}, + }, +} +ENCRYPTED_REASONING: Final = "reasoning.encrypted_content" CONVERSE_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType( { "toolSpec": { @@ -159,10 +171,27 @@ def _spend_row(identity: str) -> dict[str, JsonValue]: return rows[0] +def _only_spend_row_of(model: str) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows( + 'SELECT model_group, status, prompt_tokens, completion_tokens, api_base FROM "LiteLLM_SpendLogs"' + " WHERE model_group=%s", + (model,), + ), + lambda found: len(found) == 1, + seconds=70, + ) + return rows[0] + + def _success_row(model: str, api_base: str) -> dict[str, JsonValue]: return {"model_group": model, "status": "success", "prompt_tokens": 9, "completion_tokens": 5, "api_base": api_base} +def _bridged_success_row(model: str, wire_url: str) -> dict[str, JsonValue]: + return {**_success_row(model, f"{wire_url}{NATIVE_RESPONSES}"), "prompt_tokens": 30} + + def _delta_text(delta: ChoiceDelta, field: str) -> str: value: Final = delta.model_dump().get(field) return value if isinstance(value, str) else "" @@ -387,17 +416,234 @@ def test_json_schema_response_format_is_forwarded_natively(gateway: Gateway) -> assert _spend_row(f"chatcmpl-{marker}") == _success_row(model, f"{wire.url}{NATIVE_TARGET}") -def test_tools_while_reasoning_keep_converse(gateway: Gateway) -> None: +def test_tools_while_reasoning_are_bridged_to_native_responses(gateway: Gateway) -> None: marker: Final = uuid.uuid4().hex with wire_server(respond) as wire, gateway.scenario() as scenario: model: Final = _deployment(scenario, wire) response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high") payload: Final = _payload(response) + assert payload["id"] == f"resp_upstream_{marker}", response.text + choices: Final = payload["choices"] + assert isinstance(choices, list) and len(choices) == 1, response.text + assert choices[0]["finish_reason"] == "stop", response.text + assert choices[0]["message"]["content"] == answer(marker), response.text + body: Final = _body(_converse_request(wire, NATIVE_RESPONSES)) + assert body["model"] == GPT, body + assert body["reasoning"] == {"effort": "high"}, body + assert (body["store"], body["include"]) == (False, [ENCRYPTED_REASONING]), body + assert [(tool["type"], tool["name"], tool["parameters"]) for tool in body["tools"]] == [ + ("function", "lookup_invoice", dict(TOOL_PARAMETERS)) + ], body + assert _upstream_requests_mentioning(gateway, marker) == [], response.text + assert _only_spend_row_of(model) == _bridged_success_row(model, wire.url) + + +def test_streamed_tools_while_reasoning_are_bridged_to_native_responses(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + stream: Final = _openai_client(gateway).chat.completions.create( + model=model, + messages=_sdk_messages(marker), + tools=[SDK_TOOL], + reasoning_effort="high", + stream=True, + stream_options={"include_usage": True}, + extra_body={"cache": {"no-cache": True}}, + ) + chunks: Final = list(stream) + assert _joined(chunks, "content") == answer(marker), chunks + usage: Final = chunks[-1].usage + assert usage is not None and (usage.prompt_tokens, usage.completion_tokens) == (30, 5), chunks[-1] + body: Final = _body(_converse_request(wire, NATIVE_RESPONSES)) + assert (body["model"], body["stream"], body["reasoning"]) == (GPT, True, {"effort": "high"}), body + assert (body["store"], body["include"]) == (False, [ENCRYPTED_REASONING]), body + assert [tool["name"] for tool in body["tools"]] == ["lookup_invoice"], body + assert _only_spend_row_of(model) == _bridged_success_row(model, wire.url) + + +def _reasoning_deployment(scenario: Scenario, wire: Wire, target: str) -> str: + if target == OPENAI_CODEX: + return scenario.model(model=f"openai/{OPENAI_CODEX}", api_base=wire.url, api_key="synthetic-openai-key") + return _deployment(scenario, wire, model=f"bedrock/{target}") + + +def _message(payload: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return object_value(object_value(list_value(payload["choices"])[0])["message"]) + + +def _reasoning_turn(gateway: Gateway, model: str, marker: str) -> list[dict[str, JsonValue]]: + response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high") + return rv.ITEMS.validate_python(_message(_payload(response))["reasoning_items"]) + + +def _replay( + gateway: Gateway, model: str, marker: str, reasoning_items: Sequence[Mapping[str, JsonValue]] +) -> dict[str, JsonValue]: + history: Final[list[JsonValue]] = [ + {"role": "user", "content": "Pick an invoice."}, + {"role": "assistant", "content": "Invoice 7", "reasoning_items": [dict(item) for item in reasoning_items]}, + {"role": "user", "content": _prompt(marker)}, + ] + return _payload( + gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": history, + "tools": [dict(TOOL)], + "reasoning_effort": "high", + "cache": {"no-cache": True}, + }, + ) + ) + + +def _vendor_reasoning_item(item_id: str) -> dict[str, JsonValue]: + return { + "type": "reasoning", + "id": item_id, + "summary": [], + "encrypted_content": f"gAAAAA-vendor-{item_id.removeprefix('rs_')}", + } + + +@pytest.mark.parametrize("replay_to", [GLOBAL_GPT, GPT], ids=["to-another-inference-profile", "to-the-same-profile"]) +def test_bridged_reasoning_replays_unchanged_to_the_bedrock_model_that_minted_it( + gateway: Gateway, replay_to: str +) -> None: + marker: Final = uuid.uuid4().hex + follow_up: Final = uuid.uuid4().hex + vendor: Final = rv.ResponsesVendor() + with wire_server(vendor.respond) as wire, gateway.scenario() as scenario: + minter: Final = _deployment(scenario, wire) + replayer: Final = minter if replay_to == GPT else _reasoning_deployment(scenario, wire, replay_to) + (item,) = _reasoning_turn(gateway, minter, marker) + (minted,) = vendor.minted + assert item["id"] == minted, item + assert target_of(_only_request(wire)) == NATIVE_RESPONSES + message: Final = _message(_replay(gateway, replayer, follow_up, [item])) + assert message["content"] == rv.answer(follow_up), message + body: Final = _body(_converse_request(wire, NATIVE_RESPONSES)) + assert body["model"] == replay_to, body + assert rv.reasoning_items(body) == [_vendor_reasoning_item(minted)], body["input"] + + +@pytest.mark.parametrize( + ("minted_on", "replayed_to"), + [(GPT, OTHER_GPT), (GPT, OPENAI_CODEX), (OPENAI_CODEX, GPT)], + ids=["bedrock-to-another-bedrock-model", "bedrock-to-openai", "openai-to-bedrock"], +) +def test_reasoning_is_not_replayed_to_a_model_that_did_not_mint_it( + gateway: Gateway, minted_on: str, replayed_to: str +) -> None: + marker: Final = uuid.uuid4().hex + follow_up: Final = uuid.uuid4().hex + vendor: Final = rv.ResponsesVendor() + with wire_server(vendor.respond) as wire, gateway.scenario() as scenario: + minter: Final = _reasoning_deployment(scenario, wire, minted_on) + replayer: Final = _reasoning_deployment(scenario, wire, replayed_to) + (item,) = _reasoning_turn(gateway, minter, marker) + assert item["id"] == vendor.minted[0], item + wire.drain() + message: Final = _message(_replay(gateway, replayer, follow_up, [item])) + assert message["content"] == rv.answer(follow_up), message + body: Final = _body(_only_request(wire)) + assert body["model"] == replayed_to, body + assert rv.reasoning_items(body) == [], body["input"] + assert [entry.get("role") for entry in rv.ITEMS.validate_python(body["input"])] == [ + "user", + "assistant", + "user", + ], body["input"] + + +def test_tools_while_reasoning_with_a_provider_specific_field_keep_converse_and_reach_the_model( + gateway: Gateway, +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high", foo_bar=1) + assert _message(_payload(response))["content"] == answer(marker), response.text + body: Final = _body(_converse_request(wire)) + assert object_value(body["additionalModelRequestFields"])["foo_bar"] == 1, body + assert body["toolConfig"] == {"tools": [dict(CONVERSE_TOOL)]}, body + + +def test_tools_while_reasoning_with_a_non_boolean_store_are_refused_like_converse(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high", store="false") + assert response.status_code == 400, response.text + assert "store" in response.text, response.text + assert wire.drain() == (), response.text + + +def test_tools_while_reasoning_on_a_sigv4_deployment_are_bridged_signed_with_its_keys(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"bedrock/{GPT}", + api_key=None, + aws_access_key_id=ACCESS_KEY, + aws_secret_access_key=SECRET_KEY, + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint=wire.url, + ) + response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high") + assert _payload(response)["id"] == f"resp_upstream_{marker}", response.text + request: Final = _only_request(wire) + assert (request.method, target_of(request)) == ("POST", NATIVE_RESPONSES), request.target + _assert_sigv4_signed(request, NATIVE_RESPONSES) + assert _body(request)["reasoning"] == {"effort": "high"}, request.body + assert _only_spend_row_of(model) == _bridged_success_row(model, wire.url) + + +@pytest.mark.parametrize("suffix", ["", "/openai/v1"]) +def test_tools_while_reasoning_are_bridged_to_the_api_base_without_a_runtime_endpoint( + gateway: Gateway, suffix: str +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model_info=None, **{**GPT_DEPLOYMENT, "api_base": f"{wire.url}{suffix}"}) + response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high") + request: Final = _only_request(wire) + assert (request.method, request.target) == ("POST", NATIVE_RESPONSES), response.text + assert _payload(response)["id"] == f"resp_upstream_{marker}", response.text + assert _body(request)["reasoning"] == {"effort": "high"}, request.body + assert _only_spend_row_of(model) == _bridged_success_row(model, wire.url) + + +def test_tools_while_reasoning_effort_is_an_object_keep_converse(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + response: Final = _chat( + gateway, model, marker, tools=[dict(TOOL)], reasoning_effort={"effort": "high", "summary": "auto"} + ) + payload: Final = _payload(response) + assert payload["choices"][0]["message"]["content"] == answer(marker), response.text + assert _body(_converse_request(wire))["toolConfig"] == {"tools": [dict(CONVERSE_TOOL)]} + assert _spend_row(str(payload["id"])) == _success_row(model, f"{wire.url}{CONVERSE_TARGET}") + + +def test_tools_while_reasoning_with_a_name_openai_rejects_keep_converse(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + tool: Final = {**TOOL, "function": {**TOOL["function"], "name": "lookup invoice!"}} + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + response: Final = _chat(gateway, model, marker, tools=[tool], reasoning_effort="high") + payload: Final = _payload(response) assert payload["choices"] == [ {"finish_reason": "stop", "index": 0, "message": {"content": answer(marker), "role": "assistant"}} ], response.text body: Final = _body(_converse_request(wire)) - assert body["toolConfig"] == {"tools": [CONVERSE_TOOL]}, body + assert body["toolConfig"] == { + "tools": [{"toolSpec": {**CONVERSE_TOOL["toolSpec"], "name": "lookup_invoice_"}}] + }, body assert body["additionalModelRequestFields"] == {"reasoning": {"effort": "high"}}, body assert _spend_row(str(payload["id"])) == _success_row(model, f"{wire.url}{CONVERSE_TARGET}") diff --git a/tests/integration/sdk/test_bedrock_tools_while_reasoning_sync_sdk.py b/tests/integration/sdk/test_bedrock_tools_while_reasoning_sync_sdk.py new file mode 100644 index 00000000000..263986aea5a --- /dev/null +++ b/tests/integration/sdk/test_bedrock_tools_while_reasoning_sync_sdk.py @@ -0,0 +1,61 @@ +import uuid +from collections.abc import Mapping +from typing import Final + +import pytest +from integration._support.bedrock_runtime_peer import answer, respond, target_of +from integration._support.client import eventually +from integration._support.wire import wire_server + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.utils import Choices, ModelResponse + +_MODEL_ID: Final = "us.openai.gpt-5.6-sol" +_TOOL: Final = { + "type": "function", + "function": { + "name": "lookup_invoice", + "description": "Look up an invoice", + "parameters": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}, + }, +} + + +class _SuccessRecorder(CustomLogger): + successes: tuple[tuple[object, object], ...] = () + + def log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.successes = (*self.successes, (kwargs.get("call_type"), getattr(response_obj, "id", None))) + + +def test_sync_completion_with_tools_while_reasoning_keeps_converse_and_logs_one_success( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorder: Final = _SuccessRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire: + response: Final = litellm.completion( + model=f"bedrock/{_MODEL_ID}", + messages=[{"role": "user", "content": f"synthetic native request marker-{marker}"}], + tools=[_TOOL], + reasoning_effort="high", + api_key="synthetic-bedrock-bearer", + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint=wire.url, + num_retries=0, + ) + sent: Final = [(request.method, target_of(request)) for request in wire.drain()] + assert sent == [("POST", f"/model/{_MODEL_ID}/converse")], sent + assert isinstance(response, ModelResponse), type(response) + choice: Final = response.choices[0] + assert isinstance(choice, Choices), choice + assert choice.message.content == answer(marker), choice + logged: Final = eventually( + lambda: recorder.successes, + lambda successes: any(logged_id == response.id for _, logged_id in successes), + ) + assert logged == (("completion", response.id),), logged diff --git a/tests/unit/completion_extras/test_responses_bridge_provider_propagation.py b/tests/unit/completion_extras/test_responses_bridge_provider_propagation.py index 5aff9397747..34339b3a220 100644 --- a/tests/unit/completion_extras/test_responses_bridge_provider_propagation.py +++ b/tests/unit/completion_extras/test_responses_bridge_provider_propagation.py @@ -13,6 +13,7 @@ from litellm.completion_extras.litellm_responses_transformation.handler import ( ResponsesToCompletionBridgeHandler, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.utils import ModelResponse MODEL = "openai.gpt-5.5" @@ -193,3 +194,33 @@ async def test_acompletion_without_cache_field_is_served_from_cache(_bridged_cac assert len(_bridged_cache_edge.calls) == 1 assert [first.id, second.id] == ["resp_1", "resp_1"] + + +@pytest.mark.parametrize( + "bridged_request", + [ + {"model": "openai/gpt-5.5-pro"}, + {"model": "azure/gpt-5-pro", "api_version": "2025-04-01-preview"}, + {**_GPT_6_REQUEST, "reasoning_effort": "low"}, + ], + ids=["openai_responses_mode", "azure_responses_mode", "azure_tools_with_reasoning"], +) +async def test_async_bridged_call_is_sent_through_the_callers_client(bridged_request): + sent: Final[list[httpx.Request]] = [] + + async def handle(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response(200, json=_responses_api_body(1)) + + response = await litellm.acompletion( + **{ + "messages": [{"role": "user", "content": "hi"}], + "api_key": "x", + **bridged_request, + "api_base": "http://127.0.0.1:9", + }, + client=AsyncHTTPHandler(transport=httpx.MockTransport(handle)), + ) + + assert [(request.url.host, request.url.path.endswith("/responses")) for request in sent] == [("127.0.0.1", True)] + assert response.choices[0].message.content == "hi 1" diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 3faae2e3289..16a5e0fce60 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -2,6 +2,8 @@ import json import re +from collections.abc import Iterable, Mapping +from typing import Final import httpx import pytest @@ -24,7 +26,7 @@ from litellm.llms.bedrock.common_utils import ( bedrock_runtime_chat_completions_is_default, get_bedrock_chat_config, ) -from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler APPLICATION_INFERENCE_PROFILE_ARN = "arn:aws:bedrock:us-west-2:123412341234:application-inference-profile/a1b2c3" @@ -260,6 +262,16 @@ def _recording_client(**response_kwargs): return requests, HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle))) +def _async_recording_client(**response_kwargs): + requests: list[httpx.Request] = [] + + async def handle(request): + requests.append(request) + return httpx.Response(200, **response_kwargs) + + return requests, AsyncHTTPHandler(transport=httpx.MockTransport(handle)) + + @pytest.mark.parametrize( "model, model_path", [ @@ -549,20 +561,79 @@ def test_bearer_api_key_is_sent_as_the_authorization_header(monkeypatch): assert headers["Authorization"] == "Bearer bedrock-api-key" +FILE_SEARCH_TOOL = {"type": "file_search", "vector_store_ids": ["vs_other_team"]} + + +def _weather_tool_named(name): + return {**GET_WEATHER_TOOL, "function": {**GET_WEATHER_TOOL["function"], "name": name}} + + @pytest.mark.parametrize( "request_params, expected_route", [ - ({"tools": [GET_WEATHER_TOOL]}, "converse"), - ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}, "converse"), - ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": None}, "converse"), + ({"tools": [GET_WEATHER_TOOL]}, "responses"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}, "responses"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": None}, "responses"), ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "none"}, "chat_completions"), + ( + { + "tools": [GET_WEATHER_TOOL], + "reasoning_effort": "low", + "guardrailConfig": {"guardrailIdentifier": "gr-1"}, + }, + "converse", + ), + ({"tools": [GET_WEATHER_TOOL, FILE_SEARCH_TOOL], "reasoning_effort": "low"}, "converse"), + ({"tools": GET_WEATHER_TOOL, "reasoning_effort": "low"}, "converse"), + ({"tools": ["get_weather"], "reasoning_effort": "low"}, "converse"), + ({"tools": [{"type": "function"}], "reasoning_effort": "low"}, "converse"), + ({"tools": [_weather_tool_named("get weather!")], "reasoning_effort": "low"}, "converse"), + ({"tools": [_weather_tool_named("weather.get")], "reasoning_effort": "low"}, "converse"), + ({"tools": [_weather_tool_named("w" * 65)], "reasoning_effort": "low"}, "converse"), + ({"tools": [_weather_tool_named("w" * 64)], "reasoning_effort": "low"}, "responses"), + ({"tools": [_weather_tool_named("get-weather_2")], "reasoning_effort": "low"}, "responses"), + ( + {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"tools": [FILE_SEARCH_TOOL]}}, + "converse", + ), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"store": False}}, "responses"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"top_k": 5}}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": "store=false"}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "seed": 7}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "logit_bias": {"50256": -100}}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "web_search_options": {}}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "n": 2}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "n": 1}, "responses"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "presence_penalty": 0.5}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "presence_penalty": 0}, "responses"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "logprobs": True}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "logprobs": False}, "responses"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "prompt_cache_key": "k"}, "responses"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": {"effort": "low", "summary": "auto"}}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "store": "false"}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "store": 0}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "foo_bar": 1}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "foo_bar": None}, "responses"), + ( + { + "tools": [GET_WEATHER_TOOL], + "reasoning_effort": "low", + "model": "global.openai.gpt-6-sol", + "messages": [{"role": "user", "content": "hi"}], + "drop_params": True, + "aws_region_name": "us-east-1", + "metadata": {"team": "a"}, + "reasoning_summary": "auto", + }, + "responses", + ), ({"reasoning_effort": "low"}, "chat_completions"), ({"tools": None, "reasoning_effort": "low"}, "chat_completions"), ({"tools": [], "reasoning_effort": "low"}, "chat_completions"), ({}, "chat_completions"), ], ) -def test_gpt56_tools_need_reasoning_none_on_chat_completions(local_cost_map, request_params, expected_route): +def test_gpt56_tools_with_reasoning_bridge_to_responses(local_cost_map, request_params, expected_route): assert ( BedrockModelInfo.get_bedrock_route("chat_completions/global.openai.gpt-5.6-sol", request_params) == expected_route @@ -603,6 +674,21 @@ def test_gpt56_legacy_functions_route_like_tools(local_cost_map, request_params, ) +def test_gpt56_tools_with_reasoning_stay_on_converse_without_a_responses_endpoint(local_cost_map, monkeypatch): + model = "global.openai.gpt-6-sol" + monkeypatch.setitem( + litellm.model_cost, model, {**litellm.model_cost[model], "supported_endpoints": ["/v1/chat/completions"]} + ) + params = {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"} + assert BedrockModelInfo.get_bedrock_route(model, params) == "converse" + assert BedrockModelInfo.get_bedrock_route(model, {**params, "reasoning_effort": "none"}) == "chat_completions" + + +def test_gpt56_region_path_tools_with_reasoning_stay_on_converse(local_cost_map): + params = {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"} + assert BedrockModelInfo.get_bedrock_route("bedrock/us-west-2/global.openai.gpt-6-sol", params) == "converse" + + def test_thinking_block_goes_to_converse(local_cost_map): thinking = {"type": "enabled", "budget_tokens": 1024} assert BedrockModelInfo.get_bedrock_route("chat_completions/us.xai.grok-4.6", {"thinking": thinking}) == "converse" @@ -1055,19 +1141,407 @@ def test_gpt_oss_completion_hits_chat_completions_and_splits_reasoning(local_cos assert response.choices[0].message.content == "Hi" -def test_gpt56_tools_with_reasoning_effort_go_to_converse(local_cost_map, fake_aws_env): +RESPONSES_TOOL_CALL_JSON = { + "id": "resp_test", + "object": "response", + "created_at": 1733529600, + "status": "completed", + "output": [ + { + "type": "function_call", + "id": "fc_0", + "call_id": "call_0", + "name": "get_weather", + "arguments": '{"city": "Paris"}', + "status": "completed", + } + ], + "usage": {"input_tokens": 5, "output_tokens": 3, "total_tokens": 8}, +} + + +@pytest.mark.parametrize( + "model", ["bedrock/chat_completions/global.openai.gpt-5.6-sol", "bedrock/global.openai.gpt-6-sol"] +) +async def test_gpt56_and_newer_tools_with_reasoning_effort_bridge_to_runtime_responses( + local_cost_map, fake_aws_env, model +): + requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": model}) + response = await litellm.acompletion( + model=model, + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + temperature=0.2, + drop_params=True, + client=client, + ) + + assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/responses" + body = json.loads(requests[0].content) + assert body["model"] == model.removeprefix("bedrock/").removeprefix("chat_completions/") + assert body["reasoning"] == {"effort": "low"} + assert [tool["name"] for tool in body["tools"]] == ["get_weather"] + assert "temperature" not in body + assert body["store"] is False + assert body["include"] == ["reasoning.encrypted_content"] + assert response.object == "chat.completion" + assert response.choices[0].finish_reason == "tool_calls" + assert response.choices[0].message.tool_calls[0].function.name == "get_weather" + assert response._hidden_params["response_cost"] > 0 + + +async def test_a_provider_specific_param_keeps_converse_and_reaches_the_model(local_cost_map, fake_aws_env): + requests, client = _async_recording_client(json=CONVERSE_JSON) + await litellm.acompletion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + foo_bar=1, + client=client, + ) + + assert [request.url.raw_path for request in requests] == [b"/model/global.openai.gpt-6-sol/converse"] + assert json.loads(requests[0].content)["additionalModelRequestFields"]["foo_bar"] == 1 + + +async def test_tools_with_reasoning_bridge_stores_only_when_the_caller_asks(local_cost_map, fake_aws_env): + requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) + await litellm.acompletion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + store=True, + client=client, + ) + + body = json.loads(requests[0].content) + assert body["store"] is True + assert "include" not in body + + +async def test_tools_with_reasoning_and_a_param_the_bridge_drops_fail_like_converse_without_drop_params( + local_cost_map, fake_aws_env +): + requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) + with pytest.raises(litellm.UnsupportedParamsError, match="seed"): + await litellm.acompletion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + seed=7, + client=client, + ) + + assert requests == [] + + +@pytest.mark.parametrize( + "runtime_endpoint_param, runtime_endpoint_env, expected_host", + [ + ("https://vpce-param.example.com", None, "vpce-param.example.com"), + (None, "https://vpce-env.example.com", "vpce-env.example.com"), + (None, None, "proxy.example.com"), + ], +) +async def test_tools_with_reasoning_bridge_goes_to_the_host_converse_would( + local_cost_map, fake_aws_env, monkeypatch, runtime_endpoint_param, runtime_endpoint_env, expected_host +): + if runtime_endpoint_env: + monkeypatch.setenv("AWS_BEDROCK_RUNTIME_ENDPOINT", runtime_endpoint_env) + params = { + "model": "bedrock/global.openai.gpt-6-sol", + "messages": [{"role": "user", "content": "weather in Paris"}], + "reasoning_effort": "low", + "api_base": "https://proxy.example.com", + "aws_bedrock_runtime_endpoint": runtime_endpoint_param, + } + converse_requests, converse_client = _async_recording_client(json=CONVERSE_JSON) + await litellm.acompletion(**params, tools=[_weather_tool_named("get weather!")], client=converse_client) + bridge_requests, bridge_client = _async_recording_client( + json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"} + ) + await litellm.acompletion(**params, tools=[GET_WEATHER_TOOL], client=bridge_client) + + assert converse_requests[0].url.raw_path.endswith(b"/converse") + assert bridge_requests[0].url.path == "/openai/v1/responses" + assert bridge_requests[0].url.host == converse_requests[0].url.host == expected_host + + +async def test_tools_with_reasoning_and_a_name_openai_rejects_keep_converse_and_its_name_mapping( + local_cost_map, fake_aws_env +): + requests, client = _async_recording_client(json=CONVERSE_JSON) + await litellm.acompletion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[_weather_tool_named("get weather!")], + reasoning_effort="low", + client=client, + ) + + assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse") + assert json.loads(requests[0].content)["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather_" + + +async def test_a_caller_param_named_like_the_bridge_flag_neither_crashes_nor_steers_the_route( + local_cost_map, fake_aws_env +): + requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) + response = await litellm.acompletion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + _litellm_responses_api_bridge_allowed=False, + client=client, + ) + + assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/responses" + assert response.choices[0].message.tool_calls[0].function.name == "get_weather" + + +async def test_tools_with_reasoning_bridge_keeps_the_reasoning_summary_alias(local_cost_map, fake_aws_env): + requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) + await litellm.acompletion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + extra_body={"reasoningSummary": "auto"}, + client=client, + ) + + assert json.loads(requests[0].content)["reasoning"] == {"effort": "low", "summary": "auto"} + + +def test_sync_tools_with_reasoning_stay_on_converse(local_cost_map, fake_aws_env): requests, client = _recording_client(json=CONVERSE_JSON) - response = litellm.completion( - model="bedrock/chat_completions/global.openai.gpt-5.6-sol", - messages=[{"role": "user", "content": "hello"}], + litellm.completion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], tools=[GET_WEATHER_TOOL], reasoning_effort="low", client=client, ) - assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-5.6-sol/converse") - assert json.loads(requests[0].content)["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather" - assert response.choices[0].message.content == "ok" + assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse") + + +async def test_tools_with_an_object_reasoning_effort_stay_on_converse_like_main(local_cost_map, fake_aws_env): + requests, client = _async_recording_client(json=CONVERSE_JSON) + await litellm.acompletion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort={"effort": "low", "summary": "auto"}, + client=client, + ) + + assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse") + + +def _responses_reply_with_reasoning(encrypted_content: str, model: str) -> dict: + return { + **RESPONSES_TOOL_CALL_JSON, + "model": model, + "output": [ + {"type": "reasoning", "id": "rs_0", "encrypted_content": encrypted_content, "summary": []}, + *RESPONSES_TOOL_CALL_JSON["output"], + ], + } + + +def _responses_stream_with_reasoning(encrypted_content: str, model: str) -> bytes: + completed: Final = {"type": "response.completed", "response": _responses_reply_with_reasoning(encrypted_content, model)} + return f"event: response.completed\ndata: {json.dumps(completed)}\n\n".encode() + + +async def _bedrock_turn( + history: list, model: str = "global.openai.gpt-6-sol", **response_kwargs +) -> tuple[list[httpx.Request], object]: + requests, client = _async_recording_client(**response_kwargs) + response = await litellm.acompletion( + model=f"bedrock/{model}", + messages=history, + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + stream="text/event-stream" in response_kwargs.get("headers", {}).values(), + client=client, + ) + return requests, response + + +async def _openai_turn(history: list, reply: dict) -> tuple[list[httpx.Request], object]: + requests, client = _async_recording_client(json=reply) + response = await litellm.acompletion( + model="openai/gpt-5.5-pro", + api_key="x", + api_base="http://127.0.0.1:9", + messages=history, + tools=[GET_WEATHER_TOOL], + client=client, + ) + return requests, response + + +def _history_after(reasoning_items: Iterable[Mapping[str, object]]) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "weather in Paris"}, + { + "role": "assistant", + "content": None, + "reasoning_items": [dict(item) for item in reasoning_items], + "tool_calls": [ + { + "id": "call_0", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_0", "content": "14C, light rain"}, + ] + + +def _replayed_reasoning(request: httpx.Request) -> list: + return [ + item.get("encrypted_content") for item in json.loads(request.content)["input"] if item.get("type") == "reasoning" + ] + + +async def _streamed_reasoning_items(response) -> list: + deltas: Final = [chunk.choices[0].delta async for chunk in response if chunk.choices] + return [item for delta in deltas for item in getattr(delta, "reasoning_items", None) or ()] + + +@pytest.mark.parametrize("stream", [False, True], ids=["plain", "streamed"]) +async def test_reasoning_bedrock_minted_replays_to_bedrock_but_never_to_openai(local_cost_map, fake_aws_env, stream): + reply_kwargs: Final = ( + { + "content": _responses_stream_with_reasoning("bedrock-minted", "global.openai.gpt-6-sol"), + "headers": {"content-type": "text/event-stream"}, + } + if stream + else {"json": _responses_reply_with_reasoning("bedrock-minted", "global.openai.gpt-6-sol")} + ) + _, first = await _bedrock_turn([{"role": "user", "content": "weather in Paris"}], **reply_kwargs) + reasoning_items: Final = ( + await _streamed_reasoning_items(first) if stream else list(first.choices[0].message.reasoning_items) + ) + history: Final = _history_after(reasoning_items) + + to_bedrock, _ = await _bedrock_turn(history, json=RESPONSES_TOOL_CALL_JSON) + to_openai, _ = await _openai_turn(history, {**RESPONSES_TOOL_CALL_JSON, "model": "gpt-5.5-pro"}) + + assert len(reasoning_items) == 1 + assert _replayed_reasoning(to_bedrock[0]) == ["bedrock-minted"] + assert _replayed_reasoning(to_openai[0]) == [] + + +async def test_reasoning_one_bedrock_model_minted_never_replays_to_another(local_cost_map, fake_aws_env): + _, first = await _bedrock_turn( + [{"role": "user", "content": "weather in Paris"}], + json=_responses_reply_with_reasoning("gpt-6-minted", "global.openai.gpt-6-sol"), + ) + history: Final = _history_after(first.choices[0].message.reasoning_items) + + to_same_model, _ = await _bedrock_turn(history, json=RESPONSES_TOOL_CALL_JSON) + to_same_model_us_profile, _ = await _bedrock_turn( + history, model="us.openai.gpt-6-sol", json=RESPONSES_TOOL_CALL_JSON + ) + to_other_model, _ = await _bedrock_turn(history, model="us.openai.gpt-5.6-sol", json=RESPONSES_TOOL_CALL_JSON) + + assert _replayed_reasoning(to_same_model[0]) == ["gpt-6-minted"] + assert _replayed_reasoning(to_same_model_us_profile[0]) == ["gpt-6-minted"] + assert _replayed_reasoning(to_other_model[0]) == [] + + +async def test_reasoning_openai_minted_replays_to_openai_but_never_to_bedrock(local_cost_map, fake_aws_env): + _, first = await _openai_turn( + [{"role": "user", "content": "weather in Paris"}], _responses_reply_with_reasoning("openai-minted", "gpt-5.5-pro") + ) + history: Final = _history_after(first.choices[0].message.reasoning_items) + + to_openai, _ = await _openai_turn(history, {**RESPONSES_TOOL_CALL_JSON, "model": "gpt-5.5-pro"}) + to_bedrock, _ = await _bedrock_turn(history, json=RESPONSES_TOOL_CALL_JSON) + + assert _replayed_reasoning(to_openai[0]) == ["openai-minted"] + assert _replayed_reasoning(to_bedrock[0]) == [] + + +async def test_async_tools_with_reasoning_bridge_uses_the_supplied_client(local_cost_map, fake_aws_env): + requests: Final[list[httpx.Request]] = [] + + async def handle(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) + + response = await litellm.acompletion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + client=AsyncHTTPHandler(transport=httpx.MockTransport(handle)), + ) + + assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/responses" + assert response.choices[0].message.tool_calls[0].function.name == "get_weather" + + +class _FrozenCredentials: + access_key = "AKIDLEGACYCLIENT" + secret_key = "legacy-secret" + token = None + + +class _LegacyCredentials: + def get_frozen_credentials(self): + return _FrozenCredentials() + + +class _LegacyBedrockClient: + class meta: + region_name = "eu-west-1" + + def _get_credentials(self): + return _LegacyCredentials() + + +async def test_tools_with_reasoning_and_a_legacy_aws_bedrock_client_stay_on_converse(local_cost_map, fake_aws_env): + requests, client = _async_recording_client(json=CONVERSE_JSON) + await litellm.acompletion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + aws_bedrock_client=_LegacyBedrockClient(), + client=client, + ) + + assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse") + assert requests[0].url.host == "bedrock-runtime.eu-west-1.amazonaws.com" + assert "Credential=AKIDLEGACYCLIENT/" in requests[0].headers["Authorization"] + + +def test_responses_forced_onto_chat_completions_serves_tools_with_reasoning_on_converse(local_cost_map, fake_aws_env): + requests, client = _recording_client(json=CONVERSE_JSON) + litellm.responses( + model="bedrock/global.openai.gpt-6-sol", + input="weather in Paris", + tools=[{"type": "function", "name": "get_weather", "parameters": GET_WEATHER_TOOL["function"]["parameters"]}], + reasoning={"effort": "low"}, + use_chat_completions_api=True, + client=client, + ) + + assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse") + body = json.loads(requests[0].content) + assert body["additionalModelRequestFields"] == {"reasoning": {"effort": "low"}} + assert body["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather" def test_gpt56_tools_with_reasoning_none_stay_on_chat_completions(local_cost_map, fake_aws_env): @@ -1113,23 +1587,6 @@ def test_gpt_56_and_newer_completion_without_the_prefix_posts_runtime_chat_compl assert response._hidden_params["response_cost"] > 0 -def test_gpt6_without_the_prefix_tools_with_reasoning_effort_go_to_converse(local_cost_map, fake_aws_env): - requests, client = _recording_client(json=CONVERSE_JSON) - response = litellm.completion( - model="bedrock/global.openai.gpt-6-sol", - messages=[{"role": "user", "content": "hello"}], - tools=[GET_WEATHER_TOOL], - reasoning_effort="low", - client=client, - ) - - assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse") - body = json.loads(requests[0].content) - assert body["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather" - assert body["additionalModelRequestFields"]["reasoning"] == {"effort": "low"} - assert response.choices[0].message.content == "ok" - - def test_gpt6_without_the_prefix_guardrail_config_goes_to_converse(local_cost_map, fake_aws_env): guardrail = {"guardrailIdentifier": "gr-1", "guardrailVersion": "1"} requests, client = _recording_client(json=CONVERSE_JSON) diff --git a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py index 6da131f38cc..462ec36d97d 100644 --- a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py +++ b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py @@ -57,6 +57,19 @@ class TestCompleteURL: ) assert url == "https://vpce.example.com/openai/v1/responses" + @pytest.mark.parametrize( + "runtime_endpoint_param, runtime_endpoint_env", + [("https://vpce.example.com", None), (None, "https://env-vpce.example.com")], + ) + def test_api_base_wins_over_a_runtime_endpoint(self, monkeypatch, runtime_endpoint_param, runtime_endpoint_env): + if runtime_endpoint_env: + monkeypatch.setenv("AWS_BEDROCK_RUNTIME_ENDPOINT", runtime_endpoint_env) + url = _cfg().get_complete_url( + "https://proxy.example.com", + {"aws_region_name": "us-east-1", "aws_bedrock_runtime_endpoint": runtime_endpoint_param}, + ) + assert url == "https://proxy.example.com/openai/v1/responses" + class TestAuth: def test_bearer_token_is_used_when_present(self):