From e2966ab2dd7f9668f714d8c7f671ca939cc42088 Mon Sep 17 00:00:00 2001 From: Matthew Lapointe Date: Fri, 9 Oct 2026 22:05:17 -0400 Subject: [PATCH 01/14] feat(bedrock): bridge gpt-5.6+ function tools with reasoning to native /responses instead of Converse --- .../handler.py | 7 +- litellm/llms/bedrock/common_utils.py | 59 +++++- litellm/main.py | 33 ++- litellm/utils.py | 8 +- ...bedrock_chat_completions_transformation.py | 192 +++++++++++++++--- 5 files changed, 256 insertions(+), 43 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index edc32dc487a..f27e2e4eb33 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -5,7 +5,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req from collections.abc import AsyncIterable, Coroutine, Iterable from typing import TYPE_CHECKING, Any, Final, Union -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict from litellm.types.llms.openai import ResponsesAPIResponse @@ -23,6 +23,7 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict): logging_obj: "LiteLLMLoggingObj" custom_llm_provider: str encoding: object + client: ReadOnly[object] def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str: @@ -152,6 +153,7 @@ class ResponsesToCompletionBridgeHandler: logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, encoding=typed_kwargs.get("encoding"), + client=typed_kwargs.get("client"), ) def completion( @@ -186,7 +188,7 @@ class ResponsesToCompletionBridgeHandler: litellm_params=litellm_params, headers=headers, litellm_logging_obj=logging_obj, - client=kwargs.get("client"), + client=validated_kwargs["client"], ) # Set on request_data rather than passed as explicit kwargs: the spread of @@ -282,6 +284,7 @@ class ResponsesToCompletionBridgeHandler: litellm_params=litellm_params, headers=headers, litellm_logging_obj=logging_obj, + client=validated_kwargs["client"], ) except Exception as e: raise e diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 151f4858fdb..c2cf9e1f4dd 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -59,6 +59,7 @@ BedrockRoute = Literal[ "openai", "mantle", "chat_completions", + "responses", ] @@ -997,7 +998,8 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje written onto the Converse body, function tools (``tools`` or legacy ``functions``) on a model without ``supports_bedrock_runtime_chat_completions_tools_with_reasoning`` are rejected there unless - ``reasoning_effort`` is exactly ``"none"``, and a ``response_format`` goes native only as + ``reasoning_effort`` is exactly ``"none"`` (function-only ``tools`` on a model listing ``/v1/responses`` are + bridged to bedrock-runtime's native Responses API instead), and a ``response_format`` goes native only as ``{"type": "json_schema", "json_schema": ...}`` (a pydantic model is converted to that) on a model with ``supports_bedrock_runtime_chat_completions_response_format``: a schema on any other model is only honored by Converse, and every ``json_object`` form (``response_schema`` included) keeps Converse's @@ -1015,6 +1017,12 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje return True if _response_format_needs_converse(model, request_params.get("response_format")): return True + return _chat_completions_rejects_function_tools(model, request_params) and not _bridgeable_to_responses( + model, request_params + ) + + +def _chat_completions_rejects_function_tools(model: str, request_params: Mapping[str, object]) -> bool: if not (request_params.get("tools") or request_params.get("functions")): return False return ( @@ -1023,12 +1031,50 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje ) +def _function_tools_only(tools: object) -> bool: + if not isinstance(tools, (list, tuple)): + return False + try: + return all(tool.get("type") == "function" for tool in _TOOL_DICTS_ADAPTER.validate_python(tools)) + except ValidationError: + return False + + +_EXTRA_BODY_ADAPTER: Final = TypeAdapter(dict[str, object]) + + +def _extra_body_has_tools(extra_body: object) -> bool: + return isinstance(extra_body, dict) and bool(_EXTRA_BODY_ADAPTER.validate_python(extra_body).get("tools")) + + +def _bridgeable_to_responses(model: str, request_params: Mapping[str, object]) -> bool: + return ( + not request_params.get("functions") + and request_params.get("aws_bedrock_client") is None + and _function_tools_only(request_params.get("tools")) + and not _extra_body_has_tools(request_params.get("extra_body")) + and _bedrock_serves_responses(model) + ) + + +def _bedrock_serves_responses(model: str) -> bool: + region, model_id = split_bedrock_region_path(model) + return region is None and any( + _price_map_entry_lists_endpoint(_model_cost_entry(key), "/v1/responses") + for key in (model_id, f"bedrock/{model_id}") + ) + + def _chat_completions_unless_converse_needed( model: str, request_params: Mapping[str, object] | None -) -> Literal["converse", "chat_completions"]: - if request_params is not None and bedrock_request_needs_converse(model, request_params): +) -> Literal["converse", "chat_completions", "responses"]: + if request_params is None: + return "chat_completions" + if bedrock_request_needs_converse(model, request_params): return "converse" - return "chat_completions" + if not _chat_completions_rejects_function_tools(model, request_params): + return "chat_completions" + return "converse" if request_params.get("responses_api_bridge_allowed") is False else "responses" def bedrock_route_for_request( @@ -1474,8 +1520,9 @@ class BedrockModelInfo(BaseLLMModelInfo): Grok and GPT 5.6 and newer go to bedrock-runtime's native OpenAI Chat Completions by default (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in; ``request_params`` (the caller's chat params) sends such a request to Converse when it needs a - feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family - model stays on Converse without the prefix. + feature only Converse serves, to ``responses`` (the chat-to-Responses bridge onto bedrock-runtime's + native ``/openai/v1/responses``) when it carries function tools with reasoning on, and ``converse/`` + pins a model to Converse. Every other OpenAI-family model stays on Converse without the prefix. """ route_mappings: dict[ str, diff --git a/litellm/main.py b/litellm/main.py index b80819354f0..6cb01580888 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5723,7 +5723,11 @@ def completion( "allowed_openai_params": allowed_openai_params, "base_model": base_model, } - optional_params = get_optional_params(**optional_param_args, **non_default_params) + optional_params = get_optional_params( + **optional_param_args, + **non_default_params, + responses_api_bridge_allowed=not skip_responses_api_bridge, + ) processed_non_default_params: Final = pre_process_non_default_params( model=model, passed_params=optional_param_args, @@ -5878,11 +5882,32 @@ def completion( api_base=api_base, ) + request_params: Final = MappingProxyType( + _OPTIONAL_PARAMS_ADAPTER.validate_python( + { + **optional_param_args, + **non_default_params, + "responses_api_bridge_allowed": not skip_responses_api_bridge, + } + ) + ) + bedrock_bridges_to_responses: Final = ( + custom_llm_provider == "bedrock" + and bedrock_route_for_request( + model, + request_params, + _ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(kwargs.get("additional_drop_params") or ()), + ) + == "responses" + ) + # Use base_model (the true underlying model) for Azure model-type # detection when the deployment name differs from the model name. _azure_detection_model: Final = base_model or model - if responses_api_model_info.get("mode") == "responses" and not skip_responses_api_bridge: + if ( + responses_api_model_info.get("mode") == "responses" or bedrock_bridges_to_responses + ) and not skip_responses_api_bridge: from litellm.completion_extras import responses_api_bridge optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params) @@ -5950,9 +5975,7 @@ def completion( optional_params=optional_params, organization=organization, provider_config=provider_config, - request_params=MappingProxyType( - _OPTIONAL_PARAMS_ADAPTER.validate_python({**optional_param_args, **non_default_params}) - ), + request_params=request_params, shared_session=shared_session, stream=stream, temperature=temperature, diff --git a/litellm/utils.py b/litellm/utils.py index e7a171f56c6..37b35f954c3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4522,6 +4522,7 @@ def get_optional_params( store: bool | None = None, prompt_cache_key: str | None = None, base_model: str | None = None, + responses_api_bridge_allowed: bool = True, **kwargs: object, ): drop_params = normalize_drop_params(drop_params) # rebind-ok: config and DB deployments pass "true" as a string @@ -4532,6 +4533,7 @@ def get_optional_params( # non_default_params / _check_valid_arg — it's a routing hint, not an # OpenAI param. passed_params.pop("base_model", None) + passed_params.pop("responses_api_bridge_allowed") provider_config: BaseConfig | None = None if custom_llm_provider is not None and custom_llm_provider in [provider.value for provider in LlmProviders]: provider_config = ProviderConfigManager.get_provider_chat_config( @@ -4583,7 +4585,11 @@ def get_optional_params( ) bedrock_route: Final = ( - _bedrock_route_for_request(model, passed_params, additional_drop_params) + _bedrock_route_for_request( + model, + {**passed_params, "responses_api_bridge_allowed": responses_api_bridge_allowed}, + additional_drop_params, + ) if custom_llm_provider == "bedrock" else None ) diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 1a952966514..50393b2f354 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -2,6 +2,7 @@ import json import re +from typing import Final import httpx import pytest @@ -24,7 +25,7 @@ from litellm.llms.bedrock.common_utils import ( bedrock_runtime_chat_completions_is_default, get_bedrock_chat_config, ) -from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler APPLICATION_INFERENCE_PROFILE_ARN = "arn:aws:bedrock:us-west-2:123412341234:application-inference-profile/a1b2c3" @@ -549,20 +550,39 @@ def test_bearer_api_key_is_sent_as_the_authorization_header(monkeypatch): assert headers["Authorization"] == "Bearer bedrock-api-key" +FILE_SEARCH_TOOL = {"type": "file_search", "vector_store_ids": ["vs_other_team"]} + + @pytest.mark.parametrize( "request_params, expected_route", [ - ({"tools": [GET_WEATHER_TOOL]}, "converse"), - ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}, "converse"), - ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": None}, "converse"), + ({"tools": [GET_WEATHER_TOOL]}, "responses"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}, "responses"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": None}, "responses"), ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "none"}, "chat_completions"), + ( + { + "tools": [GET_WEATHER_TOOL], + "reasoning_effort": "low", + "guardrailConfig": {"guardrailIdentifier": "gr-1"}, + }, + "converse", + ), + ({"tools": [GET_WEATHER_TOOL, FILE_SEARCH_TOOL], "reasoning_effort": "low"}, "converse"), + ({"tools": GET_WEATHER_TOOL, "reasoning_effort": "low"}, "converse"), + ({"tools": ["get_weather"], "reasoning_effort": "low"}, "converse"), + ( + {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"tools": [FILE_SEARCH_TOOL]}}, + "converse", + ), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"store": False}}, "responses"), ({"reasoning_effort": "low"}, "chat_completions"), ({"tools": None, "reasoning_effort": "low"}, "chat_completions"), ({"tools": [], "reasoning_effort": "low"}, "chat_completions"), ({}, "chat_completions"), ], ) -def test_gpt56_tools_need_reasoning_none_on_chat_completions(local_cost_map, request_params, expected_route): +def test_gpt56_tools_with_reasoning_bridge_to_responses(local_cost_map, request_params, expected_route): assert ( BedrockModelInfo.get_bedrock_route("chat_completions/global.openai.gpt-5.6-sol", request_params) == expected_route @@ -603,6 +623,21 @@ def test_gpt56_legacy_functions_route_like_tools(local_cost_map, request_params, ) +def test_gpt56_tools_with_reasoning_stay_on_converse_without_a_responses_endpoint(local_cost_map, monkeypatch): + model = "global.openai.gpt-6-sol" + monkeypatch.setitem( + litellm.model_cost, model, {**litellm.model_cost[model], "supported_endpoints": ["/v1/chat/completions"]} + ) + params = {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"} + assert BedrockModelInfo.get_bedrock_route(model, params) == "converse" + assert BedrockModelInfo.get_bedrock_route(model, {**params, "reasoning_effort": "none"}) == "chat_completions" + + +def test_gpt56_region_path_tools_with_reasoning_stay_on_converse(local_cost_map): + params = {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"} + assert BedrockModelInfo.get_bedrock_route("bedrock/us-west-2/global.openai.gpt-6-sol", params) == "converse" + + def test_thinking_block_goes_to_converse(local_cost_map): thinking = {"type": "enabled", "budget_tokens": 1024} assert BedrockModelInfo.get_bedrock_route("chat_completions/us.xai.grok-4.6", {"thinking": thinking}) == "converse" @@ -1055,19 +1090,135 @@ def test_gpt_oss_completion_hits_chat_completions_and_splits_reasoning(local_cos assert response.choices[0].message.content == "Hi" -def test_gpt56_tools_with_reasoning_effort_go_to_converse(local_cost_map, fake_aws_env): - requests, client = _recording_client(json=CONVERSE_JSON) +RESPONSES_TOOL_CALL_JSON = { + "id": "resp_test", + "object": "response", + "created_at": 1733529600, + "status": "completed", + "output": [ + { + "type": "function_call", + "id": "fc_0", + "call_id": "call_0", + "name": "get_weather", + "arguments": '{"city": "Paris"}', + "status": "completed", + } + ], + "usage": {"input_tokens": 5, "output_tokens": 3, "total_tokens": 8}, +} + + +@pytest.mark.parametrize( + "model", ["bedrock/chat_completions/global.openai.gpt-5.6-sol", "bedrock/global.openai.gpt-6-sol"] +) +def test_gpt56_and_newer_tools_with_reasoning_effort_bridge_to_runtime_responses(local_cost_map, fake_aws_env, model): + requests, client = _recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": model}) response = litellm.completion( - model="bedrock/chat_completions/global.openai.gpt-5.6-sol", - messages=[{"role": "user", "content": "hello"}], + model=model, + messages=[{"role": "user", "content": "weather in Paris"}], tools=[GET_WEATHER_TOOL], reasoning_effort="low", + temperature=0.2, + drop_params=True, client=client, ) - assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-5.6-sol/converse") - assert json.loads(requests[0].content)["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather" - assert response.choices[0].message.content == "ok" + assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/responses" + body = json.loads(requests[0].content) + assert body["model"] == model.removeprefix("bedrock/").removeprefix("chat_completions/") + assert body["reasoning"] == {"effort": "low"} + assert [tool["name"] for tool in body["tools"]] == ["get_weather"] + assert "temperature" not in body + assert response.object == "chat.completion" + assert response.choices[0].finish_reason == "tool_calls" + assert response.choices[0].message.tool_calls[0].function.name == "get_weather" + assert response._hidden_params["response_cost"] > 0 + + +def test_tools_with_reasoning_bridge_keeps_the_reasoning_summary_alias(local_cost_map, fake_aws_env): + requests, client = _recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) + litellm.completion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + extra_body={"reasoningSummary": "auto"}, + client=client, + ) + + assert json.loads(requests[0].content)["reasoning"] == {"effort": "low", "summary": "auto"} + + +async def test_async_tools_with_reasoning_bridge_uses_the_supplied_client(local_cost_map, fake_aws_env): + requests: Final[list[httpx.Request]] = [] + + async def handle(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) + + response = await litellm.acompletion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + client=AsyncHTTPHandler(transport=httpx.MockTransport(handle)), + ) + + assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/responses" + assert response.choices[0].message.tool_calls[0].function.name == "get_weather" + + +class _FrozenCredentials: + access_key = "AKIDLEGACYCLIENT" + secret_key = "legacy-secret" + token = None + + +class _LegacyCredentials: + def get_frozen_credentials(self): + return _FrozenCredentials() + + +class _LegacyBedrockClient: + class meta: + region_name = "eu-west-1" + + def _get_credentials(self): + return _LegacyCredentials() + + +def test_tools_with_reasoning_and_a_legacy_aws_bedrock_client_stay_on_converse(local_cost_map, fake_aws_env): + requests, client = _recording_client(json=CONVERSE_JSON) + litellm.completion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + aws_bedrock_client=_LegacyBedrockClient(), + client=client, + ) + + assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse") + assert requests[0].url.host == "bedrock-runtime.eu-west-1.amazonaws.com" + assert "Credential=AKIDLEGACYCLIENT/" in requests[0].headers["Authorization"] + + +def test_responses_forced_onto_chat_completions_serves_tools_with_reasoning_on_converse(local_cost_map, fake_aws_env): + requests, client = _recording_client(json=CONVERSE_JSON) + litellm.responses( + model="bedrock/global.openai.gpt-6-sol", + input="weather in Paris", + tools=[{"type": "function", "name": "get_weather", "parameters": GET_WEATHER_TOOL["function"]["parameters"]}], + reasoning={"effort": "low"}, + use_chat_completions_api=True, + client=client, + ) + + assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse") + body = json.loads(requests[0].content) + assert body["additionalModelRequestFields"] == {"reasoning": {"effort": "low"}} + assert body["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather" def test_gpt56_tools_with_reasoning_none_stay_on_chat_completions(local_cost_map, fake_aws_env): @@ -1113,23 +1264,6 @@ def test_gpt_56_and_newer_completion_without_the_prefix_posts_runtime_chat_compl assert response._hidden_params["response_cost"] > 0 -def test_gpt6_without_the_prefix_tools_with_reasoning_effort_go_to_converse(local_cost_map, fake_aws_env): - requests, client = _recording_client(json=CONVERSE_JSON) - response = litellm.completion( - model="bedrock/global.openai.gpt-6-sol", - messages=[{"role": "user", "content": "hello"}], - tools=[GET_WEATHER_TOOL], - reasoning_effort="low", - client=client, - ) - - assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse") - body = json.loads(requests[0].content) - assert body["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather" - assert body["additionalModelRequestFields"]["reasoning"] == {"effort": "low"} - assert response.choices[0].message.content == "ok" - - def test_gpt6_without_the_prefix_guardrail_config_goes_to_converse(local_cost_map, fake_aws_env): guardrail = {"guardrailIdentifier": "gr-1", "guardrailVersion": "1"} requests, client = _recording_client(json=CONVERSE_JSON) From 30c668700be4e280b30ef8800e4c57c5cfbf1706 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:36:21 -0700 Subject: [PATCH 02/14] fix(bedrock): keep Converse for tool names OpenAI rejects, prefix the bridge flag, honor aws_bedrock_runtime_endpoint on Responses --- litellm/llms/bedrock/common_utils.py | 18 +++++++- .../llms/bedrock/responses/transformation.py | 16 +++---- litellm/main.py | 4 +- litellm/utils.py | 6 +-- ...t_bedrock_runtime_chat_completions_wire.py | 46 +++++++++++++++++-- ...bedrock_chat_completions_transformation.py | 40 ++++++++++++++++ .../test_bedrock_openai_responses.py | 7 +++ 7 files changed, 117 insertions(+), 20 deletions(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index c2cf9e1f4dd..30360d77722 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1031,13 +1031,27 @@ def _chat_completions_rejects_function_tools(model: str, request_params: Mapping ) +class _NamedFunction(TypedDict): + name: ReadOnly[str] + + +class _FunctionTool(TypedDict): + type: ReadOnly[Literal["function"]] + function: ReadOnly[_NamedFunction] + + +_FUNCTION_TOOLS_ADAPTER: Final = TypeAdapter(tuple[_FunctionTool, ...]) +_OPENAI_FUNCTION_NAME: Final = re.compile(r"[a-zA-Z0-9_-]{1,64}") + + def _function_tools_only(tools: object) -> bool: if not isinstance(tools, (list, tuple)): return False try: - return all(tool.get("type") == "function" for tool in _TOOL_DICTS_ADAPTER.validate_python(tools)) + validated: Final = _FUNCTION_TOOLS_ADAPTER.validate_python(tools) except ValidationError: return False + return all(_OPENAI_FUNCTION_NAME.fullmatch(tool["function"]["name"]) for tool in validated) _EXTRA_BODY_ADAPTER: Final = TypeAdapter(dict[str, object]) @@ -1074,7 +1088,7 @@ def _chat_completions_unless_converse_needed( return "converse" if not _chat_completions_rejects_function_tools(model, request_params): return "chat_completions" - return "converse" if request_params.get("responses_api_bridge_allowed") is False else "responses" + return "converse" if request_params.get("_litellm_responses_api_bridge_allowed") is False else "responses" def bedrock_route_for_request( diff --git a/litellm/llms/bedrock/responses/transformation.py b/litellm/llms/bedrock/responses/transformation.py index 086d1211835..256b1b86132 100644 --- a/litellm/llms/bedrock/responses/transformation.py +++ b/litellm/llms/bedrock/responses/transformation.py @@ -222,17 +222,13 @@ class BedrockOpenAIResponsesConfig(BaseAWSLLM, OpenAIResponsesAPIConfig): api_base: str | None, litellm_params: dict, # mutable-ok: signature fixed by the BaseResponsesAPIConfig override contract ) -> str: - region: Final = self._get_aws_region_name(optional_params=litellm_params, model=None) - override: Final = ( - api_base - or litellm_params.get("aws_bedrock_runtime_endpoint") - or get_secret_str("AWS_BEDROCK_RUNTIME_ENDPOINT") + runtime_endpoint: Final = litellm_params.get("aws_bedrock_runtime_endpoint") + _, destination = self.get_runtime_endpoint( + api_base=api_base or None, + aws_bedrock_runtime_endpoint=runtime_endpoint if isinstance(runtime_endpoint, str) else None, + aws_region_name=self._get_aws_region_name(optional_params=litellm_params, model=None), ) - # Partition-aware: bedrock-runtime is amazonaws.com.cn in China, and other - # suffixes in GovCloud/ISO, so defer to the shared endpoint builder. - host: Final = ( - override or self._select_default_endpoint_url(endpoint_type="runtime", aws_region_name=region) - ).rstrip("/") + host: Final = destination.rstrip("/") base: Final = next( (host[: -len(suffix)] for suffix in BEDROCK_RUNTIME_OPENAI_BASE_SUFFIXES if host.endswith(suffix)), host, diff --git a/litellm/main.py b/litellm/main.py index 6cb01580888..b3143ebea97 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5726,7 +5726,7 @@ def completion( optional_params = get_optional_params( **optional_param_args, **non_default_params, - responses_api_bridge_allowed=not skip_responses_api_bridge, + _litellm_responses_api_bridge_allowed=not skip_responses_api_bridge, ) processed_non_default_params: Final = pre_process_non_default_params( model=model, @@ -5887,7 +5887,7 @@ def completion( { **optional_param_args, **non_default_params, - "responses_api_bridge_allowed": not skip_responses_api_bridge, + "_litellm_responses_api_bridge_allowed": not skip_responses_api_bridge, } ) ) diff --git a/litellm/utils.py b/litellm/utils.py index 37b35f954c3..d639a60e88c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4522,7 +4522,7 @@ def get_optional_params( store: bool | None = None, prompt_cache_key: str | None = None, base_model: str | None = None, - responses_api_bridge_allowed: bool = True, + _litellm_responses_api_bridge_allowed: bool = True, **kwargs: object, ): drop_params = normalize_drop_params(drop_params) # rebind-ok: config and DB deployments pass "true" as a string @@ -4533,7 +4533,7 @@ def get_optional_params( # non_default_params / _check_valid_arg — it's a routing hint, not an # OpenAI param. passed_params.pop("base_model", None) - passed_params.pop("responses_api_bridge_allowed") + passed_params.pop("_litellm_responses_api_bridge_allowed") provider_config: BaseConfig | None = None if custom_llm_provider is not None and custom_llm_provider in [provider.value for provider in LlmProviders]: provider_config = ProviderConfigManager.get_provider_chat_config( @@ -4587,7 +4587,7 @@ def get_optional_params( bedrock_route: Final = ( _bedrock_route_for_request( model, - {**passed_params, "responses_api_bridge_allowed": responses_api_bridge_allowed}, + {**passed_params, "_litellm_responses_api_bridge_allowed": _litellm_responses_api_bridge_allowed}, additional_drop_params, ) if custom_llm_provider == "bedrock" diff --git a/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py b/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py index c8f9e5c1f5e..b1f61697405 100644 --- a/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py +++ b/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py @@ -8,7 +8,7 @@ from urllib.parse import quote import httpx import openai import pytest -from integration._support.bedrock_runtime_peer import answer, respond, target_of +from integration._support.bedrock_runtime_peer import NATIVE_RESPONSES, answer, respond, target_of from integration._support.client import Gateway, Scenario, eventually from integration._support.database import read_rows from integration._support.sigv4 import signature @@ -159,6 +159,19 @@ def _spend_row(identity: str) -> dict[str, JsonValue]: return rows[0] +def _only_spend_row_of(model: str) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows( + 'SELECT model_group, status, prompt_tokens, completion_tokens, api_base FROM "LiteLLM_SpendLogs"' + " WHERE model_group=%s", + (model,), + ), + lambda found: len(found) == 1, + seconds=70, + ) + return rows[0] + + def _success_row(model: str, api_base: str) -> dict[str, JsonValue]: return {"model_group": model, "status": "success", "prompt_tokens": 9, "completion_tokens": 5, "api_base": api_base} @@ -387,17 +400,44 @@ def test_json_schema_response_format_is_forwarded_natively(gateway: Gateway) -> assert _spend_row(f"chatcmpl-{marker}") == _success_row(model, f"{wire.url}{NATIVE_TARGET}") -def test_tools_while_reasoning_keep_converse(gateway: Gateway) -> None: +def test_tools_while_reasoning_are_bridged_to_native_responses(gateway: Gateway) -> None: marker: Final = uuid.uuid4().hex with wire_server(respond) as wire, gateway.scenario() as scenario: model: Final = _deployment(scenario, wire) response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high") payload: Final = _payload(response) + assert payload["id"] == f"resp_upstream_{marker}", response.text + choices: Final = payload["choices"] + assert isinstance(choices, list) and len(choices) == 1, response.text + assert choices[0]["finish_reason"] == "stop", response.text + assert choices[0]["message"]["content"] == answer(marker), response.text + body: Final = _body(_converse_request(wire, NATIVE_RESPONSES)) + assert body["model"] == GPT, body + assert body["reasoning"] == {"effort": "high"}, body + assert [(tool["type"], tool["name"], tool["parameters"]) for tool in body["tools"]] == [ + ("function", "lookup_invoice", dict(TOOL_PARAMETERS)) + ], body + assert _upstream_requests_mentioning(gateway, marker) == [], response.text + assert _only_spend_row_of(model) == { + **_success_row(model, f"{wire.url}{NATIVE_RESPONSES}"), + "prompt_tokens": 30, + } + + +def test_tools_while_reasoning_with_a_name_openai_rejects_keep_converse(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + tool: Final = {**TOOL, "function": {**TOOL["function"], "name": "lookup invoice!"}} + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + response: Final = _chat(gateway, model, marker, tools=[tool], reasoning_effort="high") + payload: Final = _payload(response) assert payload["choices"] == [ {"finish_reason": "stop", "index": 0, "message": {"content": answer(marker), "role": "assistant"}} ], response.text body: Final = _body(_converse_request(wire)) - assert body["toolConfig"] == {"tools": [CONVERSE_TOOL]}, body + assert body["toolConfig"] == { + "tools": [{"toolSpec": {**CONVERSE_TOOL["toolSpec"], "name": "lookup_invoice_"}}] + }, body assert body["additionalModelRequestFields"] == {"reasoning": {"effort": "high"}}, body assert _spend_row(str(payload["id"])) == _success_row(model, f"{wire.url}{CONVERSE_TARGET}") diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 50393b2f354..bf95210354e 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -553,6 +553,10 @@ def test_bearer_api_key_is_sent_as_the_authorization_header(monkeypatch): FILE_SEARCH_TOOL = {"type": "file_search", "vector_store_ids": ["vs_other_team"]} +def _weather_tool_named(name): + return {**GET_WEATHER_TOOL, "function": {**GET_WEATHER_TOOL["function"], "name": name}} + + @pytest.mark.parametrize( "request_params, expected_route", [ @@ -571,6 +575,12 @@ FILE_SEARCH_TOOL = {"type": "file_search", "vector_store_ids": ["vs_other_team"] ({"tools": [GET_WEATHER_TOOL, FILE_SEARCH_TOOL], "reasoning_effort": "low"}, "converse"), ({"tools": GET_WEATHER_TOOL, "reasoning_effort": "low"}, "converse"), ({"tools": ["get_weather"], "reasoning_effort": "low"}, "converse"), + ({"tools": [{"type": "function"}], "reasoning_effort": "low"}, "converse"), + ({"tools": [_weather_tool_named("get weather!")], "reasoning_effort": "low"}, "converse"), + ({"tools": [_weather_tool_named("weather.get")], "reasoning_effort": "low"}, "converse"), + ({"tools": [_weather_tool_named("w" * 65)], "reasoning_effort": "low"}, "converse"), + ({"tools": [_weather_tool_named("w" * 64)], "reasoning_effort": "low"}, "responses"), + ({"tools": [_weather_tool_named("get-weather_2")], "reasoning_effort": "low"}, "responses"), ( {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"tools": [FILE_SEARCH_TOOL]}}, "converse", @@ -1136,6 +1146,36 @@ def test_gpt56_and_newer_tools_with_reasoning_effort_bridge_to_runtime_responses assert response._hidden_params["response_cost"] > 0 +def test_tools_with_reasoning_and_a_name_openai_rejects_keep_converse_and_its_name_mapping(local_cost_map, fake_aws_env): + requests, client = _recording_client(json=CONVERSE_JSON) + litellm.completion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[_weather_tool_named("get weather!")], + reasoning_effort="low", + client=client, + ) + + assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse") + assert json.loads(requests[0].content)["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather_" + + +def test_a_caller_param_named_like_the_bridge_flag_neither_crashes_nor_steers_the_route(local_cost_map, fake_aws_env): + requests, client = _recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) + response = litellm.completion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + responses_api_bridge_allowed=False, + _litellm_responses_api_bridge_allowed=False, + client=client, + ) + + assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/responses" + assert response.choices[0].message.tool_calls[0].function.name == "get_weather" + + def test_tools_with_reasoning_bridge_keeps_the_reasoning_summary_alias(local_cost_map, fake_aws_env): requests, client = _recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) litellm.completion( diff --git a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py index 6da131f38cc..5819f688a96 100644 --- a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py +++ b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py @@ -57,6 +57,13 @@ class TestCompleteURL: ) assert url == "https://vpce.example.com/openai/v1/responses" + def test_runtime_endpoint_param_wins_over_api_base_like_every_other_bedrock_route(self): + url = _cfg().get_complete_url( + "https://bedrock-runtime.us-east-1.amazonaws.com", + {"aws_region_name": "us-east-1", "aws_bedrock_runtime_endpoint": "https://vpce.example.com"}, + ) + assert url == "https://vpce.example.com/openai/v1/responses" + class TestAuth: def test_bearer_token_is_used_when_present(self): From cd175d0aa37e2ae02bc3c7a41d245df1bf58f092 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 15 Oct 2026 20:54:03 -0700 Subject: [PATCH 03/14] fix(bedrock): keep Converse for params the Responses bridge drops, store off by default, Converse host for bridged calls --- litellm/llms/bedrock/common_utils.py | 62 ++++++++++++++- .../llms/bedrock/responses/transformation.py | 16 ++-- litellm/main.py | 19 ++++- ...bedrock_chat_completions_transformation.py | 79 +++++++++++++++++++ .../test_bedrock_openai_responses.py | 14 +++- 5 files changed, 175 insertions(+), 15 deletions(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 0df1140f21e..f22a0ede66c 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -37,6 +37,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.request_metadata import bedrock_request_metadata_is_owned from litellm.secret_managers.main import get_secret, get_secret_str from litellm.types.llms.bedrock import AWS_AUTH_PARAM_KEYS, AwsAuthParams +from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams if TYPE_CHECKING: from litellm.llms.custom_httpx.http_handler import HTTPHandler @@ -1064,10 +1065,44 @@ def _function_tools_only(tools: object) -> bool: _EXTRA_BODY_ADAPTER: Final = TypeAdapter(dict[str, object]) +_EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES: Final = frozenset( + {*ResponsesAPIOptionalRequestParams.__annotations__, "reasoning_effort", "reasoningSummary", "reasoning_summary"} + - {"tools"} +) +_CHAT_PARAMS_THE_RESPONSES_BRIDGE_DROPS: Final = MappingProxyType( + { + "n": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "logprobs": False, + "top_logprobs": None, + "logit_bias": None, + "seed": None, + "modalities": None, + "audio": None, + "prediction": None, + "web_search_options": None, + "function_call": None, + "include_server_side_tool_invocations": None, + } +) +_ENCRYPTED_REASONING: Final = "reasoning.encrypted_content" +_INCLUDE_ADAPTER: Final = TypeAdapter(tuple[object, ...]) -def _extra_body_has_tools(extra_body: object) -> bool: - return isinstance(extra_body, dict) and bool(_EXTRA_BODY_ADAPTER.validate_python(extra_body).get("tools")) +def _extra_body_bridgeable(extra_body: object) -> bool: + if extra_body is None: + return True + return isinstance(extra_body, dict) and ( + _EXTRA_BODY_ADAPTER.validate_python(extra_body).keys() <= _EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES + ) + + +def _sets_a_chat_param_the_responses_bridge_drops(request_params: Mapping[str, object]) -> bool: + return any( + request_params.get(param) not in (None, no_op_value) + for param, no_op_value in _CHAT_PARAMS_THE_RESPONSES_BRIDGE_DROPS.items() + ) def _bridgeable_to_responses(model: str, request_params: Mapping[str, object]) -> bool: @@ -1075,7 +1110,8 @@ def _bridgeable_to_responses(model: str, request_params: Mapping[str, object]) - not request_params.get("functions") and request_params.get("aws_bedrock_client") is None and _function_tools_only(request_params.get("tools")) - and not _extra_body_has_tools(request_params.get("extra_body")) + and _extra_body_bridgeable(request_params.get("extra_body")) + and not _sets_a_chat_param_the_responses_bridge_drops(request_params) and _bedrock_serves_responses(model) ) @@ -1088,6 +1124,26 @@ def _bedrock_serves_responses(model: str) -> bool: ) +def bedrock_responses_bridge_optional_params(optional_params: Mapping[str, object]) -> Mapping[str, object]: + if optional_params.get("store") is not None: + return optional_params + include: Final = optional_params.get("include") + requested: Final = _INCLUDE_ADAPTER.validate_python(include) if isinstance(include, (list, tuple)) else () + return { + **optional_params, + "store": False, + "include": [*requested, *(() if _ENCRYPTED_REASONING in requested else (_ENCRYPTED_REASONING,))], + } + + +def bedrock_responses_bridge_api_base(api_base: str | None, aws_bedrock_runtime_endpoint: object) -> str | None: + return ( + (aws_bedrock_runtime_endpoint if isinstance(aws_bedrock_runtime_endpoint, str) else None) + or get_secret_str("AWS_BEDROCK_RUNTIME_ENDPOINT") + or api_base + ) + + def _chat_completions_unless_converse_needed( model: str, request_params: Mapping[str, object] | None ) -> Literal["converse", "chat_completions", "responses"]: diff --git a/litellm/llms/bedrock/responses/transformation.py b/litellm/llms/bedrock/responses/transformation.py index 256b1b86132..086d1211835 100644 --- a/litellm/llms/bedrock/responses/transformation.py +++ b/litellm/llms/bedrock/responses/transformation.py @@ -222,13 +222,17 @@ class BedrockOpenAIResponsesConfig(BaseAWSLLM, OpenAIResponsesAPIConfig): api_base: str | None, litellm_params: dict, # mutable-ok: signature fixed by the BaseResponsesAPIConfig override contract ) -> str: - runtime_endpoint: Final = litellm_params.get("aws_bedrock_runtime_endpoint") - _, destination = self.get_runtime_endpoint( - api_base=api_base or None, - aws_bedrock_runtime_endpoint=runtime_endpoint if isinstance(runtime_endpoint, str) else None, - aws_region_name=self._get_aws_region_name(optional_params=litellm_params, model=None), + region: Final = self._get_aws_region_name(optional_params=litellm_params, model=None) + override: Final = ( + api_base + or litellm_params.get("aws_bedrock_runtime_endpoint") + or get_secret_str("AWS_BEDROCK_RUNTIME_ENDPOINT") ) - host: Final = destination.rstrip("/") + # Partition-aware: bedrock-runtime is amazonaws.com.cn in China, and other + # suffixes in GovCloud/ISO, so defer to the shared endpoint builder. + host: Final = ( + override or self._select_default_endpoint_url(endpoint_type="runtime", aws_region_name=region) + ).rstrip("/") base: Final = next( (host[: -len(suffix)] for suffix in BEDROCK_RUNTIME_OPENAI_BASE_SUFFIXES if host.endswith(suffix)), host, diff --git a/litellm/main.py b/litellm/main.py index b5535b382f3..1cd5a76c3b0 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -118,6 +118,8 @@ from litellm.llms.base_llm.base_model_iterator import ( from litellm.llms.base_llm.chat.transformation import with_attribution_headers from litellm.llms.bedrock.common_utils import ( BedrockModelInfo, + bedrock_responses_bridge_api_base, + bedrock_responses_bridge_optional_params, bedrock_route_for_request, without_bedrock_route_prefix, ) @@ -5876,7 +5878,11 @@ def completion( ) and not skip_responses_api_bridge: from litellm.completion_extras import responses_api_bridge - optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params) + optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params( + {**bedrock_responses_bridge_optional_params(optional_params)} + if bedrock_bridges_to_responses + else optional_params + ) if isinstance(reasoning_effort, dict) and "summary" in reasoning_effort: optional_params["reasoning_effort"] = reasoning_effort @@ -5902,7 +5908,16 @@ def completion( acompletion=acompletion, logging_obj=logging, optional_params=optional_params, - litellm_params=litellm_params, + litellm_params=( + { + **litellm_params, + "api_base": bedrock_responses_bridge_api_base( + api_base, litellm_params.get("aws_bedrock_runtime_endpoint") + ), + } + if bedrock_bridges_to_responses + else litellm_params + ), timeout=timeout, client=client, # pass AsyncOpenAI, OpenAI client custom_llm_provider=custom_llm_provider, diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 4ec922e5c23..18ae5c50743 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -586,6 +586,18 @@ def _weather_tool_named(name): "converse", ), ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"store": False}}, "responses"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"top_k": 5}}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": "store=false"}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "seed": 7}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "logit_bias": {"50256": -100}}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "web_search_options": {}}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "n": 2}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "n": 1}, "responses"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "presence_penalty": 0.5}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "presence_penalty": 0}, "responses"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "logprobs": True}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "logprobs": False}, "responses"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "prompt_cache_key": "k"}, "responses"), ({"reasoning_effort": "low"}, "chat_completions"), ({"tools": None, "reasoning_effort": "low"}, "chat_completions"), ({"tools": [], "reasoning_effort": "low"}, "chat_completions"), @@ -1140,12 +1152,79 @@ def test_gpt56_and_newer_tools_with_reasoning_effort_bridge_to_runtime_responses assert body["reasoning"] == {"effort": "low"} assert [tool["name"] for tool in body["tools"]] == ["get_weather"] assert "temperature" not in body + assert body["store"] is False + assert body["include"] == ["reasoning.encrypted_content"] assert response.object == "chat.completion" assert response.choices[0].finish_reason == "tool_calls" assert response.choices[0].message.tool_calls[0].function.name == "get_weather" assert response._hidden_params["response_cost"] > 0 +def test_tools_with_reasoning_bridge_stores_only_when_the_caller_asks(local_cost_map, fake_aws_env): + requests, client = _recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) + litellm.completion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + store=True, + client=client, + ) + + body = json.loads(requests[0].content) + assert body["store"] is True + assert "include" not in body + + +def test_tools_with_reasoning_and_a_param_the_bridge_drops_fail_like_converse_without_drop_params( + local_cost_map, fake_aws_env +): + requests, client = _recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) + with pytest.raises(litellm.UnsupportedParamsError, match="seed"): + litellm.completion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + seed=7, + client=client, + ) + + assert requests == [] + + +@pytest.mark.parametrize( + "runtime_endpoint_param, runtime_endpoint_env, expected_host", + [ + ("https://vpce-param.example.com", None, "vpce-param.example.com"), + (None, "https://vpce-env.example.com", "vpce-env.example.com"), + (None, None, "proxy.example.com"), + ], +) +def test_tools_with_reasoning_bridge_goes_to_the_host_converse_would( + local_cost_map, fake_aws_env, monkeypatch, runtime_endpoint_param, runtime_endpoint_env, expected_host +): + if runtime_endpoint_env: + monkeypatch.setenv("AWS_BEDROCK_RUNTIME_ENDPOINT", runtime_endpoint_env) + params = { + "model": "bedrock/global.openai.gpt-6-sol", + "messages": [{"role": "user", "content": "weather in Paris"}], + "reasoning_effort": "low", + "api_base": "https://proxy.example.com", + "aws_bedrock_runtime_endpoint": runtime_endpoint_param, + } + converse_requests, converse_client = _recording_client(json=CONVERSE_JSON) + litellm.completion(**params, tools=[_weather_tool_named("get weather!")], client=converse_client) + bridge_requests, bridge_client = _recording_client( + json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"} + ) + litellm.completion(**params, tools=[GET_WEATHER_TOOL], client=bridge_client) + + assert converse_requests[0].url.raw_path.endswith(b"/converse") + assert bridge_requests[0].url.path == "/openai/v1/responses" + assert bridge_requests[0].url.host == converse_requests[0].url.host == expected_host + + def test_tools_with_reasoning_and_a_name_openai_rejects_keep_converse_and_its_name_mapping(local_cost_map, fake_aws_env): requests, client = _recording_client(json=CONVERSE_JSON) litellm.completion( diff --git a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py index 5819f688a96..462ec36d97d 100644 --- a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py +++ b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py @@ -57,12 +57,18 @@ class TestCompleteURL: ) assert url == "https://vpce.example.com/openai/v1/responses" - def test_runtime_endpoint_param_wins_over_api_base_like_every_other_bedrock_route(self): + @pytest.mark.parametrize( + "runtime_endpoint_param, runtime_endpoint_env", + [("https://vpce.example.com", None), (None, "https://env-vpce.example.com")], + ) + def test_api_base_wins_over_a_runtime_endpoint(self, monkeypatch, runtime_endpoint_param, runtime_endpoint_env): + if runtime_endpoint_env: + monkeypatch.setenv("AWS_BEDROCK_RUNTIME_ENDPOINT", runtime_endpoint_env) url = _cfg().get_complete_url( - "https://bedrock-runtime.us-east-1.amazonaws.com", - {"aws_region_name": "us-east-1", "aws_bedrock_runtime_endpoint": "https://vpce.example.com"}, + "https://proxy.example.com", + {"aws_region_name": "us-east-1", "aws_bedrock_runtime_endpoint": runtime_endpoint_param}, ) - assert url == "https://vpce.example.com/openai/v1/responses" + assert url == "https://proxy.example.com/openai/v1/responses" class TestAuth: From 99118c3720ab7937593b3f692d1161acacbdb82f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:23:09 -0700 Subject: [PATCH 04/14] fix(bedrock): keep sync completion() and object reasoning_effort on Converse for tools with reasoning Sync calls through the shared Responses bridge fire success callbacks more than once, and the native chat config refuses a non-string reasoning_effort that Converse drops. Also adds the audit's integration cells for streamed, SigV4, api_base and /v1/messages bridging. --- litellm/llms/bedrock/common_utils.py | 3 +- litellm/main.py | 5 +- ...rock_messages_gpt_chat_completions_wire.py | 90 +++++++++++++++++- ...t_bedrock_runtime_chat_completions_wire.py | 93 +++++++++++++++++- ...t_responses_bridge_provider_propagation.py | 31 ++++++ ...bedrock_chat_completions_transformation.py | 95 ++++++++++++++----- 6 files changed, 282 insertions(+), 35 deletions(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index f22a0ede66c..212ac18c9f2 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -11,7 +11,7 @@ import os import re import warnings from collections.abc import Iterator, Mapping, Sequence -from types import MappingProxyType +from types import MappingProxyType, NoneType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias from typing_extensions import ReadOnly, TypedDict @@ -1112,6 +1112,7 @@ def _bridgeable_to_responses(model: str, request_params: Mapping[str, object]) - and _function_tools_only(request_params.get("tools")) and _extra_body_bridgeable(request_params.get("extra_body")) and not _sets_a_chat_param_the_responses_bridge_drops(request_params) + and isinstance(request_params.get("reasoning_effort"), (str, NoneType)) and _bedrock_serves_responses(model) ) diff --git a/litellm/main.py b/litellm/main.py index 1cd5a76c3b0..93fdc31d378 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5683,10 +5683,11 @@ def completion( "allowed_openai_params": allowed_openai_params, "base_model": base_model, } + bedrock_responses_bridge_allowed: Final = acompletion is True and not skip_responses_api_bridge optional_params = get_optional_params( **optional_param_args, **non_default_params, - _litellm_responses_api_bridge_allowed=not skip_responses_api_bridge, + _litellm_responses_api_bridge_allowed=bedrock_responses_bridge_allowed, ) processed_non_default_params: Final = pre_process_non_default_params( model=model, @@ -5855,7 +5856,7 @@ def completion( { **optional_param_args, **non_default_params, - "_litellm_responses_api_bridge_allowed": not skip_responses_api_bridge, + "_litellm_responses_api_bridge_allowed": bedrock_responses_bridge_allowed, } ) ) diff --git a/tests/integration/messages_endpoint/providers/bedrock/test_bedrock_messages_gpt_chat_completions_wire.py b/tests/integration/messages_endpoint/providers/bedrock/test_bedrock_messages_gpt_chat_completions_wire.py index 00945840808..e3fb141d950 100644 --- a/tests/integration/messages_endpoint/providers/bedrock/test_bedrock_messages_gpt_chat_completions_wire.py +++ b/tests/integration/messages_endpoint/providers/bedrock/test_bedrock_messages_gpt_chat_completions_wire.py @@ -4,7 +4,15 @@ from collections.abc import Mapping from typing import Final import anthropic -from integration._support.bedrock_runtime_peer import NATIVE_CHAT, answer, body_of, marker_of, respond, target_of +from integration._support.bedrock_runtime_peer import ( + NATIVE_CHAT, + NATIVE_RESPONSES, + answer, + body_of, + marker_of, + respond, + target_of, +) from integration._support.client import Gateway, Scenario, eventually from integration._support.database import read_rows from integration._support.wire import Request, Wire, wire_server @@ -14,6 +22,7 @@ BEDROCK_MODEL: Final = "us.openai.gpt-5.6-sol" TOKEN: Final = "synthetic-bedrock-bearer" NO_CACHE: Final[Mapping[str, JsonValue]] = {"cache": {"no-cache": True}} ANTHROPIC_VERSION: Final[Mapping[str, str]] = {"anthropic-version": "2023-06-01"} +TOOL_SCHEMA: Final[dict[str, object]] = {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]} def _question(marker: str) -> str: @@ -40,6 +49,41 @@ def _native_body(wire: Wire, marker: str) -> Mapping[str, JsonValue]: return body_of(received[0]) +def _bridged_body(wire: Wire, marker: str) -> Mapping[str, JsonValue]: + received: Final = _carrying(wire, marker) + assert [(request.method, target_of(request)) for request in received] == [("POST", NATIVE_RESPONSES)] + assert received[0].headers["authorization"] == f"Bearer {TOKEN}", received[0].headers + body: Final = body_of(received[0]) + assert (body["model"], body["reasoning"], body["max_output_tokens"]) == ( + BEDROCK_MODEL, + {"effort": "medium"}, + 4096, + ), body + assert (body["store"], body["include"]) == (False, ["reasoning.encrypted_content"]), body + assert [(tool["type"], tool["name"], tool["parameters"]) for tool in _tools_of(body)] == [ + ("function", "lookup_invoice", TOOL_SCHEMA) + ], body + return body + + +def _tools_of(body: Mapping[str, JsonValue]) -> list[dict[str, JsonValue]]: + tools: Final = body["tools"] + assert isinstance(tools, list), body + return [tool for tool in tools if isinstance(tool, dict)] + + +def _bridged_spend_row(model: str) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows( + 'SELECT status, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE model_group=%s', + (model,), + ), + lambda found: len(found) == 1, + seconds=70, + ) + return rows[0] + + def _native_request(marker: str, max_tokens: int, effort: str) -> Mapping[str, JsonValue]: return { "model": BEDROCK_MODEL, @@ -122,6 +166,50 @@ def test_anthropic_sdk_stream_with_thinking_budget_is_served_by_native_chat_comp assert _spend_rows(identity, 1) == [_success_row(identity, model, cache_hit="False")] +def test_anthropic_sdk_tools_with_thinking_are_bridged_to_native_responses(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + client: Final = anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0) + message: Final = client.messages.create( + model=model, + max_tokens=4096, + thinking={"type": "enabled", "budget_tokens": 2048}, + tools=[{"name": "lookup_invoice", "description": "Look up an invoice", "input_schema": TOOL_SCHEMA}], + messages=[{"role": "user", "content": _question(marker)}], + extra_body=NO_CACHE, + ) + assert "stream" not in _bridged_body(wire, marker) + assert [(block.type, getattr(block, "text", None)) for block in message.content] == [("text", answer(marker))] + assert (message.usage.input_tokens, message.usage.output_tokens) == (30, 5), message + assert _bridged_spend_row(model) == {"status": "success", "prompt_tokens": 30, "completion_tokens": 5} + + +def test_anthropic_sdk_stream_with_tools_and_thinking_is_bridged_to_native_responses(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + client: Final = anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0) + stream: Final = client.messages.create( + model=model, + max_tokens=4096, + thinking={"type": "enabled", "budget_tokens": 2048}, + tools=[{"name": "lookup_invoice", "description": "Look up an invoice", "input_schema": TOOL_SCHEMA}], + messages=[{"role": "user", "content": _question(marker)}], + extra_body=NO_CACHE, + stream=True, + ) + events: Final = list(stream) + assert _bridged_body(wire, marker)["stream"] is True + assert events[0].type == "message_start" and events[-1].type == "message_stop", events + assert "".join( + event.delta.text + for event in events + if event.type == "content_block_delta" and event.delta.type == "text_delta" + ) == answer(marker) + assert _bridged_spend_row(model) == {"status": "success", "prompt_tokens": 30, "completion_tokens": 5} + + def test_raw_thinking_summary_reaches_native_chat_completions_as_the_plain_effort(gateway: Gateway) -> None: marker: Final = uuid.uuid4().hex with wire_server(respond) as wire, gateway.scenario() as scenario: diff --git a/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py b/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py index b1f61697405..6f8432cbfbc 100644 --- a/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py +++ b/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py @@ -13,7 +13,7 @@ from integration._support.client import Gateway, Scenario, eventually from integration._support.database import read_rows from integration._support.sigv4 import signature from integration._support.wire import Request, Wire, wire_server -from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam +from openai.types.chat import ChatCompletionChunk, ChatCompletionFunctionToolParam, ChatCompletionMessageParam from openai.types.chat.chat_completion_chunk import ChoiceDelta from pydantic import JsonValue, TypeAdapter @@ -45,6 +45,15 @@ TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType( }, } ) +SDK_TOOL: Final[ChatCompletionFunctionToolParam] = { + "type": "function", + "function": { + "name": "lookup_invoice", + "description": "Look up an invoice", + "parameters": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}, + }, +} +ENCRYPTED_REASONING: Final = "reasoning.encrypted_content" CONVERSE_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType( { "toolSpec": { @@ -176,6 +185,10 @@ def _success_row(model: str, api_base: str) -> dict[str, JsonValue]: return {"model_group": model, "status": "success", "prompt_tokens": 9, "completion_tokens": 5, "api_base": api_base} +def _bridged_success_row(model: str, wire_url: str) -> dict[str, JsonValue]: + return {**_success_row(model, f"{wire_url}{NATIVE_RESPONSES}"), "prompt_tokens": 30} + + def _delta_text(delta: ChoiceDelta, field: str) -> str: value: Final = delta.model_dump().get(field) return value if isinstance(value, str) else "" @@ -414,14 +427,84 @@ def test_tools_while_reasoning_are_bridged_to_native_responses(gateway: Gateway) body: Final = _body(_converse_request(wire, NATIVE_RESPONSES)) assert body["model"] == GPT, body assert body["reasoning"] == {"effort": "high"}, body + assert (body["store"], body["include"]) == (False, [ENCRYPTED_REASONING]), body assert [(tool["type"], tool["name"], tool["parameters"]) for tool in body["tools"]] == [ ("function", "lookup_invoice", dict(TOOL_PARAMETERS)) ], body assert _upstream_requests_mentioning(gateway, marker) == [], response.text - assert _only_spend_row_of(model) == { - **_success_row(model, f"{wire.url}{NATIVE_RESPONSES}"), - "prompt_tokens": 30, - } + assert _only_spend_row_of(model) == _bridged_success_row(model, wire.url) + + +def test_streamed_tools_while_reasoning_are_bridged_to_native_responses(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + stream: Final = _openai_client(gateway).chat.completions.create( + model=model, + messages=_sdk_messages(marker), + tools=[SDK_TOOL], + reasoning_effort="high", + stream=True, + stream_options={"include_usage": True}, + extra_body={"cache": {"no-cache": True}}, + ) + chunks: Final = list(stream) + assert _joined(chunks, "content") == answer(marker), chunks + usage: Final = chunks[-1].usage + assert usage is not None and (usage.prompt_tokens, usage.completion_tokens) == (30, 5), chunks[-1] + body: Final = _body(_converse_request(wire, NATIVE_RESPONSES)) + assert (body["model"], body["stream"], body["reasoning"]) == (GPT, True, {"effort": "high"}), body + assert (body["store"], body["include"]) == (False, [ENCRYPTED_REASONING]), body + assert [tool["name"] for tool in body["tools"]] == ["lookup_invoice"], body + assert _only_spend_row_of(model) == _bridged_success_row(model, wire.url) + + +def test_tools_while_reasoning_on_a_sigv4_deployment_are_bridged_signed_with_its_keys(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"bedrock/{GPT}", + api_key=None, + aws_access_key_id=ACCESS_KEY, + aws_secret_access_key=SECRET_KEY, + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint=wire.url, + ) + response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high") + assert _payload(response)["id"] == f"resp_upstream_{marker}", response.text + request: Final = _only_request(wire) + assert (request.method, target_of(request)) == ("POST", NATIVE_RESPONSES), request.target + _assert_sigv4_signed(request, NATIVE_RESPONSES) + assert _body(request)["reasoning"] == {"effort": "high"}, request.body + assert _only_spend_row_of(model) == _bridged_success_row(model, wire.url) + + +@pytest.mark.parametrize("suffix", ["", "/openai/v1"]) +def test_tools_while_reasoning_are_bridged_to_the_api_base_without_a_runtime_endpoint( + gateway: Gateway, suffix: str +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model_info=None, **{**GPT_DEPLOYMENT, "api_base": f"{wire.url}{suffix}"}) + response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high") + request: Final = _only_request(wire) + assert (request.method, request.target) == ("POST", NATIVE_RESPONSES), response.text + assert _payload(response)["id"] == f"resp_upstream_{marker}", response.text + assert _body(request)["reasoning"] == {"effort": "high"}, request.body + assert _only_spend_row_of(model) == _bridged_success_row(model, wire.url) + + +def test_tools_while_reasoning_effort_is_an_object_keep_converse(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + response: Final = _chat( + gateway, model, marker, tools=[dict(TOOL)], reasoning_effort={"effort": "high", "summary": "auto"} + ) + payload: Final = _payload(response) + assert payload["choices"][0]["message"]["content"] == answer(marker), response.text + assert _body(_converse_request(wire))["toolConfig"] == {"tools": [dict(CONVERSE_TOOL)]} + assert _spend_row(str(payload["id"])) == _success_row(model, f"{wire.url}{CONVERSE_TARGET}") def test_tools_while_reasoning_with_a_name_openai_rejects_keep_converse(gateway: Gateway) -> None: diff --git a/tests/unit/completion_extras/test_responses_bridge_provider_propagation.py b/tests/unit/completion_extras/test_responses_bridge_provider_propagation.py index 5aff9397747..34339b3a220 100644 --- a/tests/unit/completion_extras/test_responses_bridge_provider_propagation.py +++ b/tests/unit/completion_extras/test_responses_bridge_provider_propagation.py @@ -13,6 +13,7 @@ from litellm.completion_extras.litellm_responses_transformation.handler import ( ResponsesToCompletionBridgeHandler, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.utils import ModelResponse MODEL = "openai.gpt-5.5" @@ -193,3 +194,33 @@ async def test_acompletion_without_cache_field_is_served_from_cache(_bridged_cac assert len(_bridged_cache_edge.calls) == 1 assert [first.id, second.id] == ["resp_1", "resp_1"] + + +@pytest.mark.parametrize( + "bridged_request", + [ + {"model": "openai/gpt-5.5-pro"}, + {"model": "azure/gpt-5-pro", "api_version": "2025-04-01-preview"}, + {**_GPT_6_REQUEST, "reasoning_effort": "low"}, + ], + ids=["openai_responses_mode", "azure_responses_mode", "azure_tools_with_reasoning"], +) +async def test_async_bridged_call_is_sent_through_the_callers_client(bridged_request): + sent: Final[list[httpx.Request]] = [] + + async def handle(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response(200, json=_responses_api_body(1)) + + response = await litellm.acompletion( + **{ + "messages": [{"role": "user", "content": "hi"}], + "api_key": "x", + **bridged_request, + "api_base": "http://127.0.0.1:9", + }, + client=AsyncHTTPHandler(transport=httpx.MockTransport(handle)), + ) + + assert [(request.url.host, request.url.path.endswith("/responses")) for request in sent] == [("127.0.0.1", True)] + assert response.choices[0].message.content == "hi 1" diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 18ae5c50743..a8df46737b9 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -261,6 +261,16 @@ def _recording_client(**response_kwargs): return requests, HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle))) +def _async_recording_client(**response_kwargs): + requests: list[httpx.Request] = [] + + async def handle(request): + requests.append(request) + return httpx.Response(200, **response_kwargs) + + return requests, AsyncHTTPHandler(transport=httpx.MockTransport(handle)) + + @pytest.mark.parametrize( "model, model_path", [ @@ -598,6 +608,7 @@ def _weather_tool_named(name): ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "logprobs": True}, "converse"), ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "logprobs": False}, "responses"), ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "prompt_cache_key": "k"}, "responses"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": {"effort": "low", "summary": "auto"}}, "converse"), ({"reasoning_effort": "low"}, "chat_completions"), ({"tools": None, "reasoning_effort": "low"}, "chat_completions"), ({"tools": [], "reasoning_effort": "low"}, "chat_completions"), @@ -1134,9 +1145,11 @@ RESPONSES_TOOL_CALL_JSON = { @pytest.mark.parametrize( "model", ["bedrock/chat_completions/global.openai.gpt-5.6-sol", "bedrock/global.openai.gpt-6-sol"] ) -def test_gpt56_and_newer_tools_with_reasoning_effort_bridge_to_runtime_responses(local_cost_map, fake_aws_env, model): - requests, client = _recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": model}) - response = litellm.completion( +async def test_gpt56_and_newer_tools_with_reasoning_effort_bridge_to_runtime_responses( + local_cost_map, fake_aws_env, model +): + requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": model}) + response = await litellm.acompletion( model=model, messages=[{"role": "user", "content": "weather in Paris"}], tools=[GET_WEATHER_TOOL], @@ -1160,9 +1173,9 @@ def test_gpt56_and_newer_tools_with_reasoning_effort_bridge_to_runtime_responses assert response._hidden_params["response_cost"] > 0 -def test_tools_with_reasoning_bridge_stores_only_when_the_caller_asks(local_cost_map, fake_aws_env): - requests, client = _recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) - litellm.completion( +async def test_tools_with_reasoning_bridge_stores_only_when_the_caller_asks(local_cost_map, fake_aws_env): + requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) + await litellm.acompletion( model="bedrock/global.openai.gpt-6-sol", messages=[{"role": "user", "content": "weather in Paris"}], tools=[GET_WEATHER_TOOL], @@ -1176,12 +1189,12 @@ def test_tools_with_reasoning_bridge_stores_only_when_the_caller_asks(local_cost assert "include" not in body -def test_tools_with_reasoning_and_a_param_the_bridge_drops_fail_like_converse_without_drop_params( +async def test_tools_with_reasoning_and_a_param_the_bridge_drops_fail_like_converse_without_drop_params( local_cost_map, fake_aws_env ): - requests, client = _recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) + requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) with pytest.raises(litellm.UnsupportedParamsError, match="seed"): - litellm.completion( + await litellm.acompletion( model="bedrock/global.openai.gpt-6-sol", messages=[{"role": "user", "content": "weather in Paris"}], tools=[GET_WEATHER_TOOL], @@ -1201,7 +1214,7 @@ def test_tools_with_reasoning_and_a_param_the_bridge_drops_fail_like_converse_wi (None, None, "proxy.example.com"), ], ) -def test_tools_with_reasoning_bridge_goes_to_the_host_converse_would( +async def test_tools_with_reasoning_bridge_goes_to_the_host_converse_would( local_cost_map, fake_aws_env, monkeypatch, runtime_endpoint_param, runtime_endpoint_env, expected_host ): if runtime_endpoint_env: @@ -1213,21 +1226,23 @@ def test_tools_with_reasoning_bridge_goes_to_the_host_converse_would( "api_base": "https://proxy.example.com", "aws_bedrock_runtime_endpoint": runtime_endpoint_param, } - converse_requests, converse_client = _recording_client(json=CONVERSE_JSON) - litellm.completion(**params, tools=[_weather_tool_named("get weather!")], client=converse_client) - bridge_requests, bridge_client = _recording_client( + converse_requests, converse_client = _async_recording_client(json=CONVERSE_JSON) + await litellm.acompletion(**params, tools=[_weather_tool_named("get weather!")], client=converse_client) + bridge_requests, bridge_client = _async_recording_client( json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"} ) - litellm.completion(**params, tools=[GET_WEATHER_TOOL], client=bridge_client) + await litellm.acompletion(**params, tools=[GET_WEATHER_TOOL], client=bridge_client) assert converse_requests[0].url.raw_path.endswith(b"/converse") assert bridge_requests[0].url.path == "/openai/v1/responses" assert bridge_requests[0].url.host == converse_requests[0].url.host == expected_host -def test_tools_with_reasoning_and_a_name_openai_rejects_keep_converse_and_its_name_mapping(local_cost_map, fake_aws_env): - requests, client = _recording_client(json=CONVERSE_JSON) - litellm.completion( +async def test_tools_with_reasoning_and_a_name_openai_rejects_keep_converse_and_its_name_mapping( + local_cost_map, fake_aws_env +): + requests, client = _async_recording_client(json=CONVERSE_JSON) + await litellm.acompletion( model="bedrock/global.openai.gpt-6-sol", messages=[{"role": "user", "content": "weather in Paris"}], tools=[_weather_tool_named("get weather!")], @@ -1239,9 +1254,11 @@ def test_tools_with_reasoning_and_a_name_openai_rejects_keep_converse_and_its_na assert json.loads(requests[0].content)["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather_" -def test_a_caller_param_named_like_the_bridge_flag_neither_crashes_nor_steers_the_route(local_cost_map, fake_aws_env): - requests, client = _recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) - response = litellm.completion( +async def test_a_caller_param_named_like_the_bridge_flag_neither_crashes_nor_steers_the_route( + local_cost_map, fake_aws_env +): + requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) + response = await litellm.acompletion( model="bedrock/global.openai.gpt-6-sol", messages=[{"role": "user", "content": "weather in Paris"}], tools=[GET_WEATHER_TOOL], @@ -1255,9 +1272,9 @@ def test_a_caller_param_named_like_the_bridge_flag_neither_crashes_nor_steers_th assert response.choices[0].message.tool_calls[0].function.name == "get_weather" -def test_tools_with_reasoning_bridge_keeps_the_reasoning_summary_alias(local_cost_map, fake_aws_env): - requests, client = _recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) - litellm.completion( +async def test_tools_with_reasoning_bridge_keeps_the_reasoning_summary_alias(local_cost_map, fake_aws_env): + requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) + await litellm.acompletion( model="bedrock/global.openai.gpt-6-sol", messages=[{"role": "user", "content": "weather in Paris"}], tools=[GET_WEATHER_TOOL], @@ -1269,6 +1286,32 @@ def test_tools_with_reasoning_bridge_keeps_the_reasoning_summary_alias(local_cos assert json.loads(requests[0].content)["reasoning"] == {"effort": "low", "summary": "auto"} +def test_sync_tools_with_reasoning_stay_on_converse(local_cost_map, fake_aws_env): + requests, client = _recording_client(json=CONVERSE_JSON) + litellm.completion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + client=client, + ) + + assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse") + + +async def test_tools_with_an_object_reasoning_effort_stay_on_converse_like_main(local_cost_map, fake_aws_env): + requests, client = _async_recording_client(json=CONVERSE_JSON) + await litellm.acompletion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort={"effort": "low", "summary": "auto"}, + client=client, + ) + + assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse") + + async def test_async_tools_with_reasoning_bridge_uses_the_supplied_client(local_cost_map, fake_aws_env): requests: Final[list[httpx.Request]] = [] @@ -1307,9 +1350,9 @@ class _LegacyBedrockClient: return _LegacyCredentials() -def test_tools_with_reasoning_and_a_legacy_aws_bedrock_client_stay_on_converse(local_cost_map, fake_aws_env): - requests, client = _recording_client(json=CONVERSE_JSON) - litellm.completion( +async def test_tools_with_reasoning_and_a_legacy_aws_bedrock_client_stay_on_converse(local_cost_map, fake_aws_env): + requests, client = _async_recording_client(json=CONVERSE_JSON) + await litellm.acompletion( model="bedrock/global.openai.gpt-6-sol", messages=[{"role": "user", "content": "weather in Paris"}], tools=[GET_WEATHER_TOOL], From 6b9823ab27ca5b509d14838f88cc9bc69f483961 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:36:25 -0700 Subject: [PATCH 05/14] fix(bedrock): replay reasoning only to the model that minted it, keep Converse for a non-boolean store or a provider-specific field Encrypted reasoning a bridged Bedrock call returns is tagged with its provider and model. A replay to the same model strips the tag, and a replay to another provider or Bedrock model drops the item, so one chat history can move between OpenAI and Bedrock deployments without a 400. A non-boolean store and any top-level field litellm passes through to the provider keep Converse, which drops or forwards them as on main. The bridge-flag test no longer passes a made-up responses_api_bridge_allowed kwarg, since an unknown field like that now keeps Converse on purpose. --- .../handler.py | 6 +- .../transformation.py | 93 ++++++++-- litellm/llms/bedrock/common_utils.py | 22 +++ ..._bedrock_tools_while_reasoning_sync_sdk.py | 61 +++++++ ...bedrock_chat_completions_transformation.py | 159 +++++++++++++++++- 5 files changed, 328 insertions(+), 13 deletions(-) create mode 100644 tests/integration/sdk/test_bedrock_tools_while_reasoning_sync_sdk.py diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 8eea095cd45..3bc1f713a42 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -253,7 +253,7 @@ class ResponsesToCompletionBridgeHandler: else: if self._is_preformatted_cached_chat_stream(result): return self._apply_post_stream_processing(result, model, custom_llm_provider) - from .transformation import is_structured_output_request + from .transformation import is_structured_output_request, reasoning_origin_for completion_stream: Final = self.transformation_handler.get_model_response_iterator( streaming_response=result, @@ -262,6 +262,7 @@ class ResponsesToCompletionBridgeHandler: structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), + reasoning_origin=reasoning_origin_for(custom_llm_provider, model), ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, @@ -357,7 +358,7 @@ class ResponsesToCompletionBridgeHandler: else: if self._is_preformatted_cached_chat_stream(result): return self._apply_post_stream_processing(result, model, custom_llm_provider) - from .transformation import is_structured_output_request + from .transformation import is_structured_output_request, reasoning_origin_for completion_stream: Final = self.transformation_handler.get_model_response_iterator( streaming_response=result, @@ -366,6 +367,7 @@ class ResponsesToCompletionBridgeHandler: structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), + reasoning_origin=reasoning_origin_for(custom_llm_provider, model), ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 979caa29832..6b85c9dc64c 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -5,6 +5,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req import json import os from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass from itertools import accumulate, chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, Union, cast @@ -305,6 +306,56 @@ def _reasoning_items_from_output_items(output_items: Sequence[object]) -> tuple[ ) +_REASONING_ORIGIN_PREFIX: Final = "litellm_reasoning_origin:" +_PROVIDERS_TAGGING_REASONING_ORIGIN: Final = frozenset({"bedrock"}) + + +@dataclass(frozen=True, slots=True) +class ReasoningReplayTarget: + origin: str | None + + +def reasoning_origin_for(provider: str | None, model: str) -> str | None: + return f"{_REASONING_ORIGIN_PREFIX}{provider}/{model}:" if provider in _PROVIDERS_TAGGING_REASONING_ORIGIN else None + + +def _with_reasoning_origin(output_items: Sequence[object], origin: str | None) -> Sequence[object]: + if origin is None: + return output_items + return tuple(_reasoning_output_item_with_origin(item, origin) for item in output_items) + + +def _reasoning_output_item_with_origin(item: object, origin: str) -> object: + from openai.types.responses import ResponseReasoningItem + + if isinstance(item, ResponseReasoningItem): + return ( + item.model_copy(update={"encrypted_content": f"{origin}{item.encrypted_content}"}) + if item.encrypted_content + else item + ) + if not isinstance(item, dict): + return item + fields: Final = cast(dict[str, object], item) # cast-ok: provider JSON item, keys forwarded verbatim + encrypted_content: Final = fields.get("encrypted_content") + if fields.get("type") != "reasoning" or not isinstance(encrypted_content, str) or not encrypted_content: + return fields + return {**fields, "encrypted_content": f"{origin}{encrypted_content}"} + + +def _replayable_reasoning_item( + item: Mapping[str, object], target: ReasoningReplayTarget +) -> Mapping[str, object] | None: + encrypted_content: Final = item.get("encrypted_content") + if not isinstance(encrypted_content, str) or not encrypted_content: + return item + if target.origin is None: + return None if encrypted_content.startswith(_REASONING_ORIGIN_PREFIX) else item + if not encrypted_content.startswith(target.origin): + return None + return {**item, "encrypted_content": encrypted_content.removeprefix(target.origin)} + + _TOOL_SEARCH_ITEM_TYPES: Final = frozenset({"tool_search_call", "tool_search_output"}) _TOOL_SEARCH_REPLAY_ITEM_TYPES: Final = frozenset({"reasoning", *_TOOL_SEARCH_ITEM_TYPES}) _TOOL_SEARCH_ITEMS_KEY: Final = "tool_search_items" @@ -428,11 +479,18 @@ def _reasoning_ids(items: Sequence[Mapping[str, object]]) -> frozenset[str]: return frozenset(item_id for item_id in ids if isinstance(item_id, str)) -def _assistant_replay_items(msg: "AllMessageValues", replay_tool_search: bool) -> tuple[Mapping[str, object], ...]: +def _assistant_replay_items( + msg: "AllMessageValues", replay_tool_search: bool, replay_to: ReasoningReplayTarget | None +) -> tuple[Mapping[str, object], ...]: searched: Final = _tool_search_input_items(msg) if replay_tool_search else () searched_reasoning_ids: Final = _reasoning_ids(searched) reasoning: Final = (item for item in _reasoning_input_items(msg) if item.get("id") not in searched_reasoning_ids) - return (*searched, *reasoning) + replayable: Final = ( + reasoning + if replay_to is None + else (kept for item in reasoning if (kept := _replayable_reasoning_item(item, replay_to)) is not None) + ) + return (*searched, *replayable) def _as_chat_reasoning_items( @@ -790,12 +848,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): keep_prompt_cache_breakpoints: bool = False, replay_tool_search: bool = False, include_namespace: bool = False, + replay_to: ReasoningReplayTarget | None = None, ) -> tuple[list[object], str | None]: converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input( messages, drop_params=drop_params, replay_tool_search=replay_tool_search, include_namespace=include_namespace, + replay_to=replay_to, ) return ( converted_input_items @@ -811,6 +871,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): drop_params: bool = False, replay_tool_search: bool = False, include_namespace: bool = False, + replay_to: ReasoningReplayTarget | None = None, ) -> tuple[list[object], str | None]: input_items: Final[list[object]] = [] instructions: str | None = None @@ -892,7 +953,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) ) elif role == "assistant" and tool_calls and isinstance(tool_calls, list): - input_items.extend(_assistant_replay_items(msg, replay_tool_search)) + input_items.extend(_assistant_replay_items(msg, replay_tool_search, replay_to)) if content: input_items.append( { @@ -933,7 +994,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): raise ValueError(f"tool call not supported: {tool_call}") elif content is not None: if role == "assistant": - input_items.extend(_assistant_replay_items(msg, replay_tool_search)) + input_items.extend(_assistant_replay_items(msg, replay_tool_search, replay_to)) input_items.append( { "type": "message", @@ -944,7 +1005,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): } ) elif role == "assistant": - input_items.extend(_assistant_replay_items(msg, replay_tool_search)) + input_items.extend(_assistant_replay_items(msg, replay_tool_search, replay_to)) return input_items, instructions @@ -1065,6 +1126,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint, replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast include_namespace=True, + replay_to=ReasoningReplayTarget(origin=reasoning_origin_for(provider, model)), ) # OpenAI's Responses API rejects an empty input. For a system-only # request, carry the system message as a system-role input item instead @@ -1419,8 +1481,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) # Convert response output to choices using the static helper + tagged_output_items: Final = _with_reasoning_origin( + output_items, reasoning_origin_for(_served_provider(litellm_params), model) + ) choices: Final = self._convert_response_output_to_choices( - output_items=output_items, + output_items=tagged_output_items, handle_raw_dict_callback=self._handle_raw_dict_response_item, structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict @@ -1439,7 +1504,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): raw_response.incomplete_details.reason if raw_response.incomplete_details is not None else None ) if len(choices) == 0: - choices.append(self._build_empty_choice(output_items, incomplete_finish_reason)) + choices.append(self._build_empty_choice(tagged_output_items, incomplete_finish_reason)) else: for choice in choices: choice.finish_reason = incomplete_finish_reason @@ -1488,9 +1553,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): sync_stream: bool, json_mode: bool | None = False, structured_output: bool = False, + reasoning_origin: str | None = None, ) -> BaseModelResponseIterator: return OpenAiResponsesToChatCompletionStreamIterator( - streaming_response, sync_stream, json_mode, structured_output=structured_output + streaming_response, + sync_stream, + json_mode, + structured_output=structured_output, + reasoning_origin=reasoning_origin, ) def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]: @@ -1869,8 +1939,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): sync_stream: bool, json_mode: bool | None = False, structured_output: bool = False, + reasoning_origin: str | None = None, ): super().__init__(streaming_response, sync_stream, json_mode) + self._reasoning_origin: Final = reasoning_origin self._chat_completion_id: str | None = None self._served_service_tier: str | None = None self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state @@ -1916,6 +1988,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): def translate_responses_chunk_to_openai_stream( parsed_chunk: dict | BaseModel, tool_call_index_map: dict[int, int] | None = None, # mutable-ok: per-stream state, remapped in place + reasoning_origin: str | None = None, ) -> "ModelResponseStream": """ Translate a Responses API streaming chunk to OpenAI chat completion streaming format. @@ -2131,7 +2204,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ) terminal_reasoning_items_typed: Final = _as_chat_reasoning_items( - _reasoning_items_from_output_items(output_items) + _reasoning_items_from_output_items(_with_reasoning_origin(output_items, reasoning_origin)) ) usage = None @@ -2195,7 +2268,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): self._empty_content_chunk() if self._is_text_delta_after_structured_answer(chunk) else OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream( - chunk, tool_call_index_map=self._tool_call_index_map + chunk, tool_call_index_map=self._tool_call_index_map, reasoning_origin=self._reasoning_origin ) ) if self._structured_output and _is_message_item_done(chunk): diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 212ac18c9f2..059a90d47a0 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -27,6 +27,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError import litellm from litellm import verbose_logger +from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES, OPENAI_CHAT_COMPLETION_PARAMS from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.optional_imports import ensure_optional_import from litellm.llms.base_llm.anthropic_messages.transformation import ( @@ -38,6 +39,7 @@ from litellm.llms.bedrock.request_metadata import bedrock_request_metadata_is_ow from litellm.secret_managers.main import get_secret, get_secret_str from litellm.types.llms.bedrock import AWS_AUTH_PARAM_KEYS, AwsAuthParams from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams +from litellm.types.utils import is_litellm_owned_kwarg if TYPE_CHECKING: from litellm.llms.custom_httpx.http_handler import HTTPHandler @@ -1105,14 +1107,34 @@ def _sets_a_chat_param_the_responses_bridge_drops(request_params: Mapping[str, o ) +_CHAT_PARAM_NAMES: Final = frozenset( + {*DEFAULT_CHAT_COMPLETION_PARAM_VALUES, *OPENAI_CHAT_COMPLETION_PARAMS, "extra_body"} +) + + +def _provider_specific_param(name: str) -> bool: + return not ( + name in _CHAT_PARAM_NAMES + or name in _EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES + or name.startswith("aws_") + or is_litellm_owned_kwarg(name) + ) + + +def _passes_a_provider_specific_param(request_params: Mapping[str, object]) -> bool: + return any(value is not None and _provider_specific_param(name) for name, value in request_params.items()) + + def _bridgeable_to_responses(model: str, request_params: Mapping[str, object]) -> bool: return ( not request_params.get("functions") and request_params.get("aws_bedrock_client") is None and _function_tools_only(request_params.get("tools")) and _extra_body_bridgeable(request_params.get("extra_body")) + and not _passes_a_provider_specific_param(request_params) and not _sets_a_chat_param_the_responses_bridge_drops(request_params) and isinstance(request_params.get("reasoning_effort"), (str, NoneType)) + and isinstance(request_params.get("store"), (bool, NoneType)) and _bedrock_serves_responses(model) ) diff --git a/tests/integration/sdk/test_bedrock_tools_while_reasoning_sync_sdk.py b/tests/integration/sdk/test_bedrock_tools_while_reasoning_sync_sdk.py new file mode 100644 index 00000000000..263986aea5a --- /dev/null +++ b/tests/integration/sdk/test_bedrock_tools_while_reasoning_sync_sdk.py @@ -0,0 +1,61 @@ +import uuid +from collections.abc import Mapping +from typing import Final + +import pytest +from integration._support.bedrock_runtime_peer import answer, respond, target_of +from integration._support.client import eventually +from integration._support.wire import wire_server + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.utils import Choices, ModelResponse + +_MODEL_ID: Final = "us.openai.gpt-5.6-sol" +_TOOL: Final = { + "type": "function", + "function": { + "name": "lookup_invoice", + "description": "Look up an invoice", + "parameters": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}, + }, +} + + +class _SuccessRecorder(CustomLogger): + successes: tuple[tuple[object, object], ...] = () + + def log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.successes = (*self.successes, (kwargs.get("call_type"), getattr(response_obj, "id", None))) + + +def test_sync_completion_with_tools_while_reasoning_keeps_converse_and_logs_one_success( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorder: Final = _SuccessRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire: + response: Final = litellm.completion( + model=f"bedrock/{_MODEL_ID}", + messages=[{"role": "user", "content": f"synthetic native request marker-{marker}"}], + tools=[_TOOL], + reasoning_effort="high", + api_key="synthetic-bedrock-bearer", + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint=wire.url, + num_retries=0, + ) + sent: Final = [(request.method, target_of(request)) for request in wire.drain()] + assert sent == [("POST", f"/model/{_MODEL_ID}/converse")], sent + assert isinstance(response, ModelResponse), type(response) + choice: Final = response.choices[0] + assert isinstance(choice, Choices), choice + assert choice.message.content == answer(marker), choice + logged: Final = eventually( + lambda: recorder.successes, + lambda successes: any(logged_id == response.id for _, logged_id in successes), + ) + assert logged == (("completion", response.id),), logged diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index a8df46737b9..6750b136bd9 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -2,6 +2,7 @@ import json import re +from collections.abc import Iterable, Mapping from typing import Final import httpx @@ -609,6 +610,23 @@ def _weather_tool_named(name): ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "logprobs": False}, "responses"), ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "prompt_cache_key": "k"}, "responses"), ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": {"effort": "low", "summary": "auto"}}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "store": "false"}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "store": 0}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "foo_bar": 1}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "foo_bar": None}, "responses"), + ( + { + "tools": [GET_WEATHER_TOOL], + "reasoning_effort": "low", + "model": "global.openai.gpt-6-sol", + "messages": [{"role": "user", "content": "hi"}], + "drop_params": True, + "aws_region_name": "us-east-1", + "metadata": {"team": "a"}, + "reasoning_summary": "auto", + }, + "responses", + ), ({"reasoning_effort": "low"}, "chat_completions"), ({"tools": None, "reasoning_effort": "low"}, "chat_completions"), ({"tools": [], "reasoning_effort": "low"}, "chat_completions"), @@ -1173,6 +1191,21 @@ async def test_gpt56_and_newer_tools_with_reasoning_effort_bridge_to_runtime_res assert response._hidden_params["response_cost"] > 0 +async def test_a_provider_specific_param_keeps_converse_and_reaches_the_model(local_cost_map, fake_aws_env): + requests, client = _async_recording_client(json=CONVERSE_JSON) + await litellm.acompletion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + foo_bar=1, + client=client, + ) + + assert [request.url.raw_path for request in requests] == [b"/model/global.openai.gpt-6-sol/converse"] + assert json.loads(requests[0].content)["additionalModelRequestFields"]["foo_bar"] == 1 + + async def test_tools_with_reasoning_bridge_stores_only_when_the_caller_asks(local_cost_map, fake_aws_env): requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) await litellm.acompletion( @@ -1263,7 +1296,6 @@ async def test_a_caller_param_named_like_the_bridge_flag_neither_crashes_nor_ste messages=[{"role": "user", "content": "weather in Paris"}], tools=[GET_WEATHER_TOOL], reasoning_effort="low", - responses_api_bridge_allowed=False, _litellm_responses_api_bridge_allowed=False, client=client, ) @@ -1312,6 +1344,131 @@ async def test_tools_with_an_object_reasoning_effort_stay_on_converse_like_main( assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-6-sol/converse") +def _responses_reply_with_reasoning(encrypted_content: str, model: str) -> dict: + return { + **RESPONSES_TOOL_CALL_JSON, + "model": model, + "output": [ + {"type": "reasoning", "id": "rs_0", "encrypted_content": encrypted_content, "summary": []}, + *RESPONSES_TOOL_CALL_JSON["output"], + ], + } + + +def _responses_stream_with_reasoning(encrypted_content: str, model: str) -> bytes: + completed: Final = {"type": "response.completed", "response": _responses_reply_with_reasoning(encrypted_content, model)} + return f"event: response.completed\ndata: {json.dumps(completed)}\n\n".encode() + + +async def _bedrock_turn( + history: list, model: str = "global.openai.gpt-6-sol", **response_kwargs +) -> tuple[list[httpx.Request], object]: + requests, client = _async_recording_client(**response_kwargs) + response = await litellm.acompletion( + model=f"bedrock/{model}", + messages=history, + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + stream="text/event-stream" in response_kwargs.get("headers", {}).values(), + client=client, + ) + return requests, response + + +async def _openai_turn(history: list, reply: dict) -> tuple[list[httpx.Request], object]: + requests, client = _async_recording_client(json=reply) + response = await litellm.acompletion( + model="openai/gpt-5.5-pro", + api_key="x", + api_base="http://127.0.0.1:9", + messages=history, + tools=[GET_WEATHER_TOOL], + client=client, + ) + return requests, response + + +def _history_after(reasoning_items: Iterable[Mapping[str, object]]) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "weather in Paris"}, + { + "role": "assistant", + "content": None, + "reasoning_items": [dict(item) for item in reasoning_items], + "tool_calls": [ + { + "id": "call_0", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_0", "content": "14C, light rain"}, + ] + + +def _replayed_reasoning(request: httpx.Request) -> list: + return [ + item.get("encrypted_content") for item in json.loads(request.content)["input"] if item.get("type") == "reasoning" + ] + + +async def _streamed_reasoning_items(response) -> list: + deltas: Final = [chunk.choices[0].delta async for chunk in response if chunk.choices] + return [item for delta in deltas for item in getattr(delta, "reasoning_items", None) or ()] + + +@pytest.mark.parametrize("stream", [False, True], ids=["plain", "streamed"]) +async def test_reasoning_bedrock_minted_replays_to_bedrock_but_never_to_openai(local_cost_map, fake_aws_env, stream): + reply_kwargs: Final = ( + { + "content": _responses_stream_with_reasoning("bedrock-minted", "global.openai.gpt-6-sol"), + "headers": {"content-type": "text/event-stream"}, + } + if stream + else {"json": _responses_reply_with_reasoning("bedrock-minted", "global.openai.gpt-6-sol")} + ) + _, first = await _bedrock_turn([{"role": "user", "content": "weather in Paris"}], **reply_kwargs) + reasoning_items: Final = ( + await _streamed_reasoning_items(first) if stream else list(first.choices[0].message.reasoning_items) + ) + history: Final = _history_after(reasoning_items) + + to_bedrock, _ = await _bedrock_turn(history, json=RESPONSES_TOOL_CALL_JSON) + to_openai, _ = await _openai_turn(history, {**RESPONSES_TOOL_CALL_JSON, "model": "gpt-5.5-pro"}) + + assert len(reasoning_items) == 1 + assert _replayed_reasoning(to_bedrock[0]) == ["bedrock-minted"] + assert _replayed_reasoning(to_openai[0]) == [] + + +async def test_reasoning_one_bedrock_model_minted_never_replays_to_another(local_cost_map, fake_aws_env): + _, first = await _bedrock_turn( + [{"role": "user", "content": "weather in Paris"}], + json=_responses_reply_with_reasoning("gpt-6-minted", "global.openai.gpt-6-sol"), + ) + history: Final = _history_after(first.choices[0].message.reasoning_items) + + to_same_model, _ = await _bedrock_turn(history, json=RESPONSES_TOOL_CALL_JSON) + to_other_model, _ = await _bedrock_turn(history, model="us.openai.gpt-5.6-sol", json=RESPONSES_TOOL_CALL_JSON) + + assert _replayed_reasoning(to_same_model[0]) == ["gpt-6-minted"] + assert _replayed_reasoning(to_other_model[0]) == [] + + +async def test_reasoning_openai_minted_replays_to_openai_but_never_to_bedrock(local_cost_map, fake_aws_env): + _, first = await _openai_turn( + [{"role": "user", "content": "weather in Paris"}], _responses_reply_with_reasoning("openai-minted", "gpt-5.5-pro") + ) + history: Final = _history_after(first.choices[0].message.reasoning_items) + + to_openai, _ = await _openai_turn(history, {**RESPONSES_TOOL_CALL_JSON, "model": "gpt-5.5-pro"}) + to_bedrock, _ = await _bedrock_turn(history, json=RESPONSES_TOOL_CALL_JSON) + + assert _replayed_reasoning(to_openai[0]) == ["openai-minted"] + assert _replayed_reasoning(to_bedrock[0]) == [] + + async def test_async_tools_with_reasoning_bridge_uses_the_supplied_client(local_cost_map, fake_aws_env): requests: Final[list[httpx.Request]] = [] From ad25280f6cd76aa810f2f8ea4d443eb1e2f49a45 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 02:20:30 -0700 Subject: [PATCH 06/14] fix(bedrock): share replayed reasoning across inference profiles of one Bedrock model The reasoning origin tag now names the Bedrock base model, so reasoning minted on a global. profile replays on the us. profile of the same model. Bedrock read the replayed reasoning across profiles in a live check, while a different model still drops it. --- .../litellm_responses_transformation/transformation.py | 7 +++++-- .../test_bedrock_chat_completions_transformation.py | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 6b85c9dc64c..c297263e981 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -307,7 +307,6 @@ def _reasoning_items_from_output_items(output_items: Sequence[object]) -> tuple[ _REASONING_ORIGIN_PREFIX: Final = "litellm_reasoning_origin:" -_PROVIDERS_TAGGING_REASONING_ORIGIN: Final = frozenset({"bedrock"}) @dataclass(frozen=True, slots=True) @@ -316,7 +315,11 @@ class ReasoningReplayTarget: def reasoning_origin_for(provider: str | None, model: str) -> str | None: - return f"{_REASONING_ORIGIN_PREFIX}{provider}/{model}:" if provider in _PROVIDERS_TAGGING_REASONING_ORIGIN else None + if provider != "bedrock": + return None + from litellm.llms.bedrock.common_utils import get_bedrock_base_model + + return f"{_REASONING_ORIGIN_PREFIX}bedrock/{get_bedrock_base_model(model)}:" def _with_reasoning_origin(output_items: Sequence[object], origin: str | None) -> Sequence[object]: diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 6750b136bd9..16a5e0fce60 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -1450,9 +1450,13 @@ async def test_reasoning_one_bedrock_model_minted_never_replays_to_another(local history: Final = _history_after(first.choices[0].message.reasoning_items) to_same_model, _ = await _bedrock_turn(history, json=RESPONSES_TOOL_CALL_JSON) + to_same_model_us_profile, _ = await _bedrock_turn( + history, model="us.openai.gpt-6-sol", json=RESPONSES_TOOL_CALL_JSON + ) to_other_model, _ = await _bedrock_turn(history, model="us.openai.gpt-5.6-sol", json=RESPONSES_TOOL_CALL_JSON) assert _replayed_reasoning(to_same_model[0]) == ["gpt-6-minted"] + assert _replayed_reasoning(to_same_model_us_profile[0]) == ["gpt-6-minted"] assert _replayed_reasoning(to_other_model[0]) == [] From 88355aad919bd561a52bea3742cee0bd650be0ec Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 03:06:49 -0700 Subject: [PATCH 07/14] test(bedrock): cover reasoning replay across inference profiles, models and providers, and params that keep Converse --- ...t_bedrock_runtime_chat_completions_wire.py | 125 +++++++++++++++++- 1 file changed, 124 insertions(+), 1 deletion(-) diff --git a/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py b/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py index 6f8432cbfbc..7630b3f6b78 100644 --- a/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py +++ b/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py @@ -8,8 +8,9 @@ from urllib.parse import quote import httpx import openai import pytest +from integration._support import responses_vendor as rv from integration._support.bedrock_runtime_peer import NATIVE_RESPONSES, answer, respond, target_of -from integration._support.client import Gateway, Scenario, eventually +from integration._support.client import Gateway, Scenario, eventually, list_value, object_value from integration._support.database import read_rows from integration._support.sigv4 import signature from integration._support.wire import Request, Wire, wire_server @@ -20,6 +21,8 @@ from pydantic import JsonValue, TypeAdapter GPT: Final = "us.openai.gpt-5.6-sol" GLOBAL_GPT: Final = "global.openai.gpt-5.6-sol" GPT_OSS: Final = "openai.gpt-oss-120b-1:0" +OTHER_GPT: Final = "global.openai.gpt-6.1-sol" +OPENAI_CODEX: Final = "gpt-5.3-codex" TOKEN: Final = "synthetic-bedrock-bearer" ACCESS_KEY: Final = "AKIASYNTHETICKEY0001" SECRET_KEY: Final = "synthetic-secret-key-for-testing" @@ -459,6 +462,126 @@ def test_streamed_tools_while_reasoning_are_bridged_to_native_responses(gateway: assert _only_spend_row_of(model) == _bridged_success_row(model, wire.url) +def _reasoning_deployment(scenario: Scenario, wire: Wire, target: str) -> str: + if target == OPENAI_CODEX: + return scenario.model(model=f"openai/{OPENAI_CODEX}", api_base=wire.url, api_key="synthetic-openai-key") + return _deployment(scenario, wire, model=f"bedrock/{target}") + + +def _message(payload: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return object_value(object_value(list_value(payload["choices"])[0])["message"]) + + +def _reasoning_turn(gateway: Gateway, model: str, marker: str) -> list[dict[str, JsonValue]]: + response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high") + return rv.ITEMS.validate_python(_message(_payload(response))["reasoning_items"]) + + +def _replay( + gateway: Gateway, model: str, marker: str, reasoning_items: Sequence[Mapping[str, JsonValue]] +) -> dict[str, JsonValue]: + history: Final[list[JsonValue]] = [ + {"role": "user", "content": "Pick an invoice."}, + {"role": "assistant", "content": "Invoice 7", "reasoning_items": [dict(item) for item in reasoning_items]}, + {"role": "user", "content": _prompt(marker)}, + ] + return _payload( + gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": history, + "tools": [dict(TOOL)], + "reasoning_effort": "high", + "cache": {"no-cache": True}, + }, + ) + ) + + +def _vendor_reasoning_item(item_id: str) -> dict[str, JsonValue]: + return { + "type": "reasoning", + "id": item_id, + "summary": [], + "encrypted_content": f"gAAAAA-vendor-{item_id.removeprefix('rs_')}", + } + + +@pytest.mark.parametrize("replay_to", [GLOBAL_GPT, GPT], ids=["to-another-inference-profile", "to-the-same-profile"]) +def test_bridged_reasoning_replays_unchanged_to_the_bedrock_model_that_minted_it( + gateway: Gateway, replay_to: str +) -> None: + marker: Final = uuid.uuid4().hex + follow_up: Final = uuid.uuid4().hex + vendor: Final = rv.ResponsesVendor() + with wire_server(vendor.respond) as wire, gateway.scenario() as scenario: + minter: Final = _deployment(scenario, wire) + replayer: Final = minter if replay_to == GPT else _reasoning_deployment(scenario, wire, replay_to) + (item,) = _reasoning_turn(gateway, minter, marker) + (minted,) = vendor.minted + assert item["id"] == minted, item + assert target_of(_only_request(wire)) == NATIVE_RESPONSES + message: Final = _message(_replay(gateway, replayer, follow_up, [item])) + assert message["content"] == rv.answer(follow_up), message + body: Final = _body(_converse_request(wire, NATIVE_RESPONSES)) + assert body["model"] == replay_to, body + assert rv.reasoning_items(body) == [_vendor_reasoning_item(minted)], body["input"] + + +@pytest.mark.parametrize( + ("minted_on", "replayed_to"), + [(GPT, OTHER_GPT), (GPT, OPENAI_CODEX), (OPENAI_CODEX, GPT)], + ids=["bedrock-to-another-bedrock-model", "bedrock-to-openai", "openai-to-bedrock"], +) +def test_reasoning_is_not_replayed_to_a_model_that_did_not_mint_it( + gateway: Gateway, minted_on: str, replayed_to: str +) -> None: + marker: Final = uuid.uuid4().hex + follow_up: Final = uuid.uuid4().hex + vendor: Final = rv.ResponsesVendor() + with wire_server(vendor.respond) as wire, gateway.scenario() as scenario: + minter: Final = _reasoning_deployment(scenario, wire, minted_on) + replayer: Final = _reasoning_deployment(scenario, wire, replayed_to) + (item,) = _reasoning_turn(gateway, minter, marker) + assert item["id"] == vendor.minted[0], item + wire.drain() + message: Final = _message(_replay(gateway, replayer, follow_up, [item])) + assert message["content"] == rv.answer(follow_up), message + body: Final = _body(_only_request(wire)) + assert body["model"] == replayed_to, body + assert rv.reasoning_items(body) == [], body["input"] + assert [entry.get("role") for entry in rv.ITEMS.validate_python(body["input"])] == [ + "user", + "assistant", + "user", + ], body["input"] + + +def test_tools_while_reasoning_with_a_provider_specific_field_keep_converse_and_reach_the_model( + gateway: Gateway, +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high", foo_bar=1) + assert _message(_payload(response))["content"] == answer(marker), response.text + body: Final = _body(_converse_request(wire)) + assert object_value(body["additionalModelRequestFields"])["foo_bar"] == 1, body + assert body["toolConfig"] == {"tools": [dict(CONVERSE_TOOL)]}, body + + +def test_tools_while_reasoning_with_a_non_boolean_store_are_refused_like_converse(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high", store="false") + assert response.status_code == 400, response.text + assert "store" in response.text, response.text + assert wire.drain() == (), response.text + + def test_tools_while_reasoning_on_a_sigv4_deployment_are_bridged_signed_with_its_keys(gateway: Gateway) -> None: marker: Final = uuid.uuid4().hex with wire_server(respond) as wire, gateway.scenario() as scenario: From 71983d831de91ba35f36e7980661a326bfcba73a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 06:27:26 -0700 Subject: [PATCH 08/14] test(bedrock): cover a declaring deployment's rounded effort on the /responses bridge --- .../test_bedrock_runtime_chat_completions_wire.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py b/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py index 7630b3f6b78..709353cad6b 100644 --- a/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py +++ b/tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py @@ -462,6 +462,19 @@ def test_streamed_tools_while_reasoning_are_bridged_to_native_responses(gateway: assert _only_spend_row_of(model) == _bridged_success_row(model, wire.url) +def test_tools_while_reasoning_are_bridged_with_the_deployments_rounded_effort(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model_info={"reasoning_effort_levels": ["low", "high"]}, + **{**GPT_DEPLOYMENT, "aws_bedrock_runtime_endpoint": wire.url}, + ) + response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="medium") + assert _payload(response)["id"] == f"resp_upstream_{marker}", response.text + body: Final = _body(_converse_request(wire, NATIVE_RESPONSES)) + assert (body["model"], body["reasoning"]) == (GPT, {"effort": "high"}), body + + def _reasoning_deployment(scenario: Scenario, wire: Wire, target: str) -> str: if target == OPENAI_CODEX: return scenario.model(model=f"openai/{OPENAI_CODEX}", api_base=wire.url, api_key="synthetic-openai-key") From b66028f1a3265fe6de2717c98266a9619624cc07 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:44:25 -0700 Subject: [PATCH 09/14] fix(bedrock): keep previous_response_id, instructions, prompt and a non-boolean extra_body store on Converse --- litellm/llms/bedrock/common_utils.py | 10 +++++++--- ...st_bedrock_chat_completions_transformation.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 5fd67f80ca2..a55f9a08195 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1226,9 +1226,10 @@ def _function_tools_only(tools: object) -> bool: _EXTRA_BODY_ADAPTER: Final = TypeAdapter(dict[str, object]) +_RESPONSES_INPUTS_CHAT_HOOKS_NEVER_CHECK: Final = frozenset({"previous_response_id", "instructions", "prompt"}) _EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES: Final = frozenset( {*ResponsesAPIOptionalRequestParams.__annotations__, "reasoning_effort", "reasoningSummary", "reasoning_summary"} - - {"tools"} + - {"tools", *_RESPONSES_INPUTS_CHAT_HOOKS_NEVER_CHECK} ) _CHAT_PARAMS_THE_RESPONSES_BRIDGE_DROPS: Final = MappingProxyType( { @@ -1254,8 +1255,11 @@ _INCLUDE_ADAPTER: Final = TypeAdapter(tuple[object, ...]) def _extra_body_bridgeable(extra_body: object) -> bool: if extra_body is None: return True - return isinstance(extra_body, dict) and ( - _EXTRA_BODY_ADAPTER.validate_python(extra_body).keys() <= _EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES + if not isinstance(extra_body, dict): + return False + validated: Final = _EXTRA_BODY_ADAPTER.validate_python(extra_body) + return validated.keys() <= _EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES and isinstance( + validated.get("store", False), bool ) diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 16a5e0fce60..c60c780e7aa 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -597,6 +597,22 @@ def _weather_tool_named(name): "converse", ), ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"store": False}}, "responses"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"store": None}}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"store": "false"}}, "converse"), + ( + {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"previous_response_id": "resp_1"}}, + "converse", + ), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "previous_response_id": "resp_1"}, "converse"), + ( + {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"instructions": "Be terse."}}, + "converse", + ), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "instructions": "Be terse."}, "converse"), + ( + {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"prompt": {"id": "pmpt_1"}}}, + "converse", + ), ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"top_k": 5}}, "converse"), ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": "store=false"}, "converse"), ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "seed": 7}, "converse"), From 222a9bf8273afd4ca26b7979983fe9fe79de87be Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:34:01 -0700 Subject: [PATCH 10/14] fix(bedrock): keep provider headers, tool search results, and encrypted reasoning on the Responses bridge Bridged streams now carry the llm_provider-* response headers, a tool_reference tool result reaches Bedrock as its tool name like Converse sent it, and an extra_body include keeps reasoning.encrypted_content beside it. The Logs page Tools section now reads function calls from a Responses-shaped response, and the Converse tool search translation case pins the explicit converse route. --- .../handler.py | 10 +++ .../transformation.py | 28 ++++++- litellm/llms/bedrock/common_utils.py | 9 ++- ...t_messages_tool_search_bedrock_converse.py | 4 +- ...bedrock_chat_completions_transformation.py | 74 +++++++++++++++++++ .../sections/ToolsSection/utils.test.ts | 32 ++++++++ .../detail/sections/ToolsSection/utils.ts | 39 +++++----- 7 files changed, 173 insertions(+), 23 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 3bc1f713a42..6221af742c9 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -16,6 +16,8 @@ from typing_extensions import ReadOnly, TypedDict from litellm.types.llms.openai import ResponsesAPIResponse if TYPE_CHECKING: + import httpx + from litellm import CustomStreamWrapper, LiteLLMLoggingObj, ModelResponse @@ -32,6 +34,12 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict): client: ReadOnly[object] +def _provider_response_headers(stream: object) -> "httpx.Headers | None": + from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator + + return stream.response.headers if isinstance(stream, BaseResponsesAPIStreamingIterator) else None + + def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str: """`responses()` runs `get_llm_provider()` itself, so hand back the prefixed model `completion()` started from.""" return f"{custom_llm_provider}/{model}" @@ -269,6 +277,7 @@ class ResponsesToCompletionBridgeHandler: model=model, custom_llm_provider=custom_llm_provider, logging_obj=logging_obj, + _response_headers=_provider_response_headers(result), ) return self._apply_post_stream_processing(streamwrapper, model, custom_llm_provider) @@ -374,6 +383,7 @@ class ResponsesToCompletionBridgeHandler: model=model, custom_llm_provider=custom_llm_provider, logging_obj=logging_obj, + _response_headers=_provider_response_headers(result), ) return self._apply_post_stream_processing(streamwrapper, model, custom_llm_provider) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 2d43398a7a7..4bb7136743f 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -336,6 +336,28 @@ def reasoning_origin_for(provider: str | None, model: str) -> str | None: return f"{_REASONING_ORIGIN_PREFIX}bedrock/{get_bedrock_base_model(model)}:" +def _tool_reference_as_text(part: object) -> object: + match part: + case {"type": "tool_reference", "tool_name": str() as tool_name}: + return {"type": "text", "text": tool_name} + case _: + return part + + +def _tool_message_with_tool_references_as_text(msg: Mapping[str, object]) -> Mapping[str, object]: + content: Final = msg.get("content") + if msg.get("role") != "tool" or not isinstance(content, list): + return msg + parts: Final = cast("list[object]", content) # cast-ok: isinstance leaves the part type unknown + return {**msg, "content": [_tool_reference_as_text(part) for part in parts]} + + +def _tool_references_as_text(messages: Sequence["AllMessageValues"]) -> tuple["AllMessageValues", ...]: + return cast( # cast-ok: each message keeps its fields, and only tool content parts change + 'tuple["AllMessageValues", ...]', tuple(_tool_message_with_tool_references_as_text(msg) for msg in messages) + ) + + def _with_reasoning_origin(output_items: Sequence[object], origin: str | None) -> Sequence[object]: if origin is None: return output_items @@ -859,7 +881,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): def convert_chat_completion_messages_to_responses_api( self, - messages: list["AllMessageValues"], + messages: Sequence["AllMessageValues"], *, drop_params: bool = False, keep_prompt_cache_breakpoints: bool = False, @@ -883,7 +905,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): def _convert_chat_completion_messages_to_responses_input( self, - messages: list["AllMessageValues"], + messages: Sequence["AllMessageValues"], *, drop_params: bool = False, replay_tool_search: bool = False, @@ -1140,7 +1162,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): and supports_openai_prompt_cache_breakpoint(base_model, provider) ) converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api( - messages, + _tool_references_as_text(messages) if provider == "bedrock" else messages, drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params), keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint, replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index a55f9a08195..9bc71d8fe27 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1313,10 +1313,17 @@ def _bedrock_serves_responses(model: str) -> bool: def bedrock_responses_bridge_optional_params(optional_params: Mapping[str, object]) -> Mapping[str, object]: if optional_params.get("store") is not None: return optional_params - include: Final = optional_params.get("include") + extra_body: Final = optional_params.get("extra_body") + extra_body_params: Final = _EXTRA_BODY_ADAPTER.validate_python(extra_body) if isinstance(extra_body, dict) else {} + include: Final = extra_body_params.get("include", optional_params.get("include")) requested: Final = _INCLUDE_ADAPTER.validate_python(include) if isinstance(include, (list, tuple)) else () return { **optional_params, + **( + {"extra_body": {key: value for key, value in extra_body_params.items() if key != "include"}} + if "include" in extra_body_params + else {} + ), "store": False, "include": [*requested, *(() if _ENCRYPTED_REASONING in requested else (_ENCRYPTED_REASONING,))], } diff --git a/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_converse.py b/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_converse.py index 2fbd4e9aa20..7f5d91fec30 100644 --- a/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_converse.py +++ b/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_converse.py @@ -11,7 +11,7 @@ GPT_6_1_SOL_TOOL_SEARCH_TEST_CASE: Final = TranslationTestCase( scenario="tool_search", litellm_endpoint="/v1/messages", litellm_request={ - "model": "bedrock/global.openai.gpt-6.1-sol", + "model": "bedrock/converse/global.openai.gpt-6.1-sol", "max_tokens": 1024, "thinking": {"type": "adaptive"}, "output_config": {"effort": "low"}, @@ -139,7 +139,7 @@ GPT_6_1_SOL_TOOL_SEARCH_TEST_CASE: Final = TranslationTestCase( "id": ANY, "type": "message", "role": "assistant", - "model": "bedrock/global.openai.gpt-6.1-sol", + "model": "bedrock/converse/global.openai.gpt-6.1-sol", "stop_sequence": None, "usage": {"input_tokens": 100, "output_tokens": 15}, "content": [{"type": "text", "text": "The web search tool is loaded and ready to use."}], diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index c60c780e7aa..894138447f8 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -1238,6 +1238,80 @@ async def test_tools_with_reasoning_bridge_stores_only_when_the_caller_asks(loca assert "include" not in body +@pytest.mark.parametrize( + "include_param", + [ + {"include": ["file_search_call.results"]}, + {"extra_body": {"include": ["file_search_call.results"]}}, + ], +) +async def test_tools_with_reasoning_bridge_keeps_encrypted_reasoning_beside_a_requested_include( + local_cost_map, fake_aws_env, include_param +): + requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) + await litellm.acompletion( + model="bedrock/global.openai.gpt-6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + client=client, + **include_param, + ) + + body = json.loads(requests[0].content) + assert body["store"] is False + assert body["include"] == ["file_search_call.results", "reasoning.encrypted_content"] + + +async def test_tools_with_reasoning_bridge_sends_a_tool_reference_result_as_its_tool_name_like_converse( + local_cost_map, fake_aws_env +): + requests, client = _async_recording_client(json={**RESPONSES_TOOL_CALL_JSON, "model": "global.openai.gpt-6-sol"}) + await litellm.acompletion( + model="bedrock/global.openai.gpt-6-sol", + messages=[ + {"role": "user", "content": "find a web tool"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "ToolSearch", "arguments": "{}"}} + ], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [{"type": "tool_reference", "tool_name": "WebSearch"}], + }, + ], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + client=client, + ) + + body = json.loads(requests[0].content) + tool_output = next(item for item in body["input"] if item["type"] == "function_call_output") + assert tool_output["output"] == [{"type": "input_text", "text": "WebSearch"}] + + +@pytest.mark.parametrize("stream", [False, True], ids=["plain", "streamed"]) +async def test_tools_with_reasoning_bridge_returns_the_provider_response_headers(local_cost_map, fake_aws_env, stream): + reply_kwargs: Final = ( + { + "content": _responses_stream_with_reasoning("bedrock-minted", "global.openai.gpt-6-sol"), + "headers": {"content-type": "text/event-stream", "x-amzn-requestid": "req-1"}, + } + if stream + else { + "json": _responses_reply_with_reasoning("bedrock-minted", "global.openai.gpt-6-sol"), + "headers": {"x-amzn-requestid": "req-1"}, + } + ) + _, response = await _bedrock_turn([{"role": "user", "content": "weather in Paris"}], **reply_kwargs) + + assert response._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-1" + + async def test_tools_with_reasoning_and_a_param_the_bridge_drops_fail_like_converse_without_drop_params( local_cost_map, fake_aws_env ): diff --git a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts index 78372fe56d9..74e9a1d1f33 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts +++ b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.test.ts @@ -335,6 +335,38 @@ describe("ToolsSection utils", () => { expect(result[0].called).toBe(true); expect(result[0].callData?.arguments).toEqual({ location: "Tokyo" }); }); + + it("should parse tool calls from a Responses API response output", () => { + const log: Partial = { + request_id: "test-responses-1", + proxy_server_request: { + tools: [ + { type: "function", function: { name: "get_weather", description: "Get current weather" } }, + { type: "function", function: { name: "get_time", description: "Get current time" } }, + ], + }, + response: { + object: "response", + output: [ + { type: "reasoning", summary: [] }, + { + type: "function_call", + call_id: "call_abc", + name: "get_weather", + arguments: '{"location": "Paris"}', + }, + ], + }, + } as any; + + const result = parseToolsFromLog(log as LogEntry); + + expect(result.map((tool) => [tool.name, tool.called])).toEqual([ + ["get_weather", true], + ["get_time", false], + ]); + expect(result[0].callData).toEqual({ id: "call_abc", name: "get_weather", arguments: { location: "Paris" } }); + }); }); describe("hasTools", () => { diff --git a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts index 5e76a09c155..789fa4f0f88 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts +++ b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts @@ -80,31 +80,36 @@ function extractToolCallsFromResponse(log: LogEntry): ToolCall[] { return responseData.tool_calls; } + // Responses API format: response.output[].type === "function_call" + if (Array.isArray(responseData.output)) { + const toolCalls = toolCallsFromOutput(responseData.output); + if (toolCalls.length > 0) return toolCalls; + } + // Realtime API format: response.results[].response.output[].type === "function_call" if (Array.isArray(responseData.results)) { - const toolCalls: ToolCall[] = []; - for (const result of responseData.results) { - if (result.type === "response.done" && result.response?.output) { - for (const item of result.response.output) { - if (item.type === "function_call") { - toolCalls.push({ - id: item.call_id || "", - type: "function", - function: { - name: item.name || "", - arguments: item.arguments || "{}", - }, - }); - } - } - } - } + const toolCalls = responseData.results + .filter((result: any) => result.type === "response.done" && Array.isArray(result.response?.output)) + .flatMap((result: any) => toolCallsFromOutput(result.response.output)); if (toolCalls.length > 0) return toolCalls; } return []; } +function toolCallsFromOutput(output: any[]): ToolCall[] { + return output + .filter((item: any) => item.type === "function_call") + .map((item: any) => ({ + id: item.call_id || "", + type: "function", + function: { + name: item.name || "", + arguments: item.arguments || "{}", + }, + })); +} + /** * Parse safe JSON with fallback */ From 29f49eb3a29c7fcdcd133dd0bd5c728483f2207c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 00:06:49 -0700 Subject: [PATCH 11/14] test(bedrock): expect the converse deployment's own key in the tool search translation case The case now calls bedrock/converse/global.openai.gpt-6.1-sol, whose deployment authenticates with synthetic-agent-key, so the expected provider header follows it --- .../tool_search/test_messages_tool_search_bedrock_converse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_converse.py b/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_converse.py index 7f5d91fec30..055571ff8c2 100644 --- a/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_converse.py +++ b/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_converse.py @@ -54,7 +54,7 @@ GPT_6_1_SOL_TOOL_SEARCH_TEST_CASE: Final = TranslationTestCase( "cache": {"no-cache": True}, }, expected_provider_endpoint="/model/global.openai.gpt-6.1-sol/converse", - expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_headers={"authorization": "Bearer synthetic-agent-key", "content-type": "application/json"}, expected_provider_request={ "additionalModelRequestFields": {"reasoning": {"effort": "low"}}, "inferenceConfig": {"maxTokens": 1024}, From 229dbaffbb82dd72c22a3274ade476a163c0f76f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:24:30 -0700 Subject: [PATCH 12/14] fix(bedrock): keep streams bridged back from Responses working and keep a non-list include on Converse A chat stream that the Responses bridge routes back to chat for a provider with no Responses config has no httpx response, so reading provider headers from it raised and the stream failed. A top-level or extra_body include that is not a list now keeps Converse, so it answers the way main does --- .../handler.py | 11 ++-- litellm/llms/bedrock/common_utils.py | 9 ++-- ...itellm_responses_transformation_handler.py | 51 +++++++++++++++++++ ...bedrock_chat_completions_transformation.py | 11 ++++ 4 files changed, 75 insertions(+), 7 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 6221af742c9..6b1f53fda51 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -11,13 +11,12 @@ from typing import ( cast, # noqa: TID251 # optional_params reaches the bridge as an untyped dict ) +import httpx from typing_extensions import ReadOnly, TypedDict from litellm.types.llms.openai import ResponsesAPIResponse if TYPE_CHECKING: - import httpx - from litellm import CustomStreamWrapper, LiteLLMLoggingObj, ModelResponse @@ -34,10 +33,14 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict): client: ReadOnly[object] -def _provider_response_headers(stream: object) -> "httpx.Headers | None": +def _provider_response_headers(stream: object) -> httpx.Headers | None: from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator - return stream.response.headers if isinstance(stream, BaseResponsesAPIStreamingIterator) else None + match stream: + case BaseResponsesAPIStreamingIterator(response=httpx.Response(headers=headers)): + return headers + case _: + return None def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str: diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 9bc71d8fe27..d6e2c9e987b 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1258,8 +1258,10 @@ def _extra_body_bridgeable(extra_body: object) -> bool: if not isinstance(extra_body, dict): return False validated: Final = _EXTRA_BODY_ADAPTER.validate_python(extra_body) - return validated.keys() <= _EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES and isinstance( - validated.get("store", False), bool + return ( + validated.keys() <= _EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES + and isinstance(validated.get("store", False), bool) + and isinstance(validated.get("include"), (list, tuple, NoneType)) ) @@ -1298,6 +1300,7 @@ def _bridgeable_to_responses(model: str, request_params: Mapping[str, object]) - and not _sets_a_chat_param_the_responses_bridge_drops(request_params) and isinstance(request_params.get("reasoning_effort"), (str, NoneType)) and isinstance(request_params.get("store"), (bool, NoneType)) + and isinstance(request_params.get("include"), (list, tuple, NoneType)) and _bedrock_serves_responses(model) ) @@ -1316,7 +1319,7 @@ def bedrock_responses_bridge_optional_params(optional_params: Mapping[str, objec extra_body: Final = optional_params.get("extra_body") extra_body_params: Final = _EXTRA_BODY_ADAPTER.validate_python(extra_body) if isinstance(extra_body, dict) else {} include: Final = extra_body_params.get("include", optional_params.get("include")) - requested: Final = _INCLUDE_ADAPTER.validate_python(include) if isinstance(include, (list, tuple)) else () + requested: Final = _INCLUDE_ADAPTER.validate_python(include or ()) return { **optional_params, **( diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py index c5d7ca96a21..1f07e7f1d15 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py @@ -10,6 +10,9 @@ from litellm.completion_extras.litellm_responses_transformation.handler import ( ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.responses.litellm_completion_transformation.streaming_iterator import ( + LiteLLMCompletionStreamingIterator, +) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ModelResponse @@ -266,6 +269,54 @@ def test_completion_streams_completed_model_response(): ) + +def _chat_stream_behind_responses() -> LiteLLMCompletionStreamingIterator: + return LiteLLMCompletionStreamingIterator( + model="claude-haiku-5-5", + litellm_custom_stream_wrapper=litellm.completion( + model="anthropic/claude-haiku-5-5", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + stream=True, + ), + request_input="hi", + responses_api_request={}, + custom_llm_provider="anthropic", + ) + + +def test_completion_streams_a_chat_stream_bridged_back_from_responses(): + bridge = ResponsesToCompletionBridgeHandler() + + with ( + patch.object( + bridge.transformation_handler, + "transform_request", + return_value={"model": "claude-haiku-5-5", "input": "hi"}, + ), + patch("litellm.responses", return_value=_chat_stream_behind_responses()), + ): + result = bridge.completion(**_bridge_kwargs(stream=True)) + + assert "".join(chunk.choices[0].delta.content or "" for chunk in result) == "pong" + + +@pytest.mark.asyncio +async def test_acompletion_streams_a_chat_stream_bridged_back_from_responses(): + bridge = ResponsesToCompletionBridgeHandler() + + with ( + patch.object( + bridge.transformation_handler, + "transform_request", + return_value={"model": "claude-haiku-5-5", "input": "hi"}, + ), + patch("litellm.aresponses", new=AsyncMock(return_value=_chat_stream_behind_responses())), + ): + result = await bridge.acompletion(**_bridge_kwargs(stream=True)) + + assert "".join([chunk.choices[0].delta.content or "" async for chunk in result]) == "pong" + _PROVIDER_NATIVE_MODEL_CASES = [ ("perplexity", "perplexity/kimi-k3", "perplexity/kimi-k3"), ("perplexity", "openai/gpt-5.2", "openai/gpt-5.2"), diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 894138447f8..6f8ea48876c 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -615,6 +615,17 @@ def _weather_tool_named(name): ), ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"top_k": 5}}, "converse"), ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": "store=false"}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "include": ["file_search_call.results"]}, "responses"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "include": "file_search_call.results"}, "converse"), + ( + {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"include": ["file_search_call.results"]}}, + "responses", + ), + ( + {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"include": "file_search_call.results"}}, + "converse", + ), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"include": {"a": 1}}}, "converse"), ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "seed": 7}, "converse"), ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "logit_bias": {"50256": -100}}, "converse"), ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "web_search_options": {}}, "converse"), From bde41798087ff4ec7f0696d8a0e25240d5fddb65 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:56:57 -0700 Subject: [PATCH 13/14] refactor(bedrock): move the responses bridge choice out of completion so basedpyright can analyze it again The main merge pushed completion past basedpyright's complexity limit, which made every name in it read as unused. The Bedrock bridge params and the reasoning summary handling now live in helpers in bedrock/common_utils.py and utils.py, and translate_responses_chunk_to_openai_stream documents its reasoning_origin argument --- .../transformation.py | 1 + litellm/llms/bedrock/common_utils.py | 23 ++++++++ litellm/main.py | 54 ++++++------------- litellm/utils.py | 17 +++++- 4 files changed, 57 insertions(+), 38 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 4bb7136743f..62e1ec070e2 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -2050,6 +2050,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): Args: parsed_chunk: Dict containing the Responses API event chunk tool_call_index_map: Per-stream output_index -> sequential tool_call index map + reasoning_origin: Origin tag stamped on streamed encrypted reasoning, so replays reach only that origin Returns: ModelResponseStream: OpenAI-formatted streaming chunk diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 806ebf51e95..aad966952e4 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1362,6 +1362,18 @@ def bedrock_responses_bridge_api_base(api_base: str | None, aws_bedrock_runtime_ ) +def bedrock_responses_bridge_call_params( + optional_params: Mapping[str, object], litellm_params: Mapping[str, object], api_base: str | None +) -> tuple[Mapping[str, object], Mapping[str, object]]: + return ( + bedrock_responses_bridge_optional_params(optional_params), + { + **litellm_params, + "api_base": bedrock_responses_bridge_api_base(api_base, litellm_params.get("aws_bedrock_runtime_endpoint")), + }, + ) + + def _chat_completions_unless_converse_needed( model: str, request_params: Mapping[str, object] | None ) -> Literal["converse", "chat_completions", "responses"]: @@ -1388,6 +1400,17 @@ def bedrock_route_for_request( ) +def bedrock_request_bridges_to_responses( + custom_llm_provider: str | None, + model: str, + request_params: Mapping[str, object], + additional_drop_params: Sequence[str] | None, +) -> bool: + if custom_llm_provider != "bedrock": + return False + return bedrock_route_for_request(model, request_params, additional_drop_params) == "responses" + + def strip_bedrock_throughput_suffix(model: str) -> str: """Strip throughput tier suffixes and context window suffixes from Bedrock model names.""" import re diff --git a/litellm/main.py b/litellm/main.py index 0e5cfbc0ec7..74b4930b00d 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -54,6 +54,7 @@ from litellm.utils import ( peek_reasoning_summary_aliases, reasoning_effort_with_summary_alias, strip_reasoning_summary_aliases_from_optional_params, + with_reasoning_summary_in_effort, ) if TYPE_CHECKING: @@ -122,8 +123,8 @@ from litellm.llms.base_llm.chat.transformation import with_attribution_headers from litellm.llms.bedrock.common_utils import ( BedrockModelInfo, NativeOutputFallback, - bedrock_responses_bridge_api_base, - bedrock_responses_bridge_optional_params, + bedrock_request_bridges_to_responses, + bedrock_responses_bridge_call_params, bedrock_route_for_request, deployment_model_info, without_bedrock_native_output_opt_in, @@ -6120,14 +6121,11 @@ def completion( } ) ) - bedrock_bridges_to_responses: Final = ( - custom_llm_provider == "bedrock" - and bedrock_route_for_request( - model, - request_params, - _ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(kwargs.get("additional_drop_params") or ()), - ) - == "responses" + bedrock_bridges_to_responses: Final = bedrock_request_bridges_to_responses( + custom_llm_provider, + model, + request_params, + _ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(kwargs.get("additional_drop_params") or ()), ) # Use base_model (the true underlying model) for Azure model-type @@ -6139,25 +6137,14 @@ def completion( ) and not skip_responses_api_bridge: from litellm.completion_extras import responses_api_bridge - optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params( - {**bedrock_responses_bridge_optional_params(optional_params)} + bridge_optional_params, bridge_litellm_params = ( + bedrock_responses_bridge_call_params(optional_params, litellm_params, api_base) if bedrock_bridges_to_responses - else optional_params + else (optional_params, litellm_params) + ) + stripped_optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params( + bridge_optional_params ) - - if isinstance(resolved_reasoning_effort, dict) and "summary" in resolved_reasoning_effort: - optional_params["reasoning_effort"] = resolved_reasoning_effort - elif rs_val is not None: - eff: Final = optional_params.get("reasoning_effort", resolved_reasoning_effort) - if isinstance(eff, dict): - optional_params["reasoning_effort"] = {**eff, "summary": rs_val} - elif eff is not None: - optional_params["reasoning_effort"] = { - "effort": eff, - "summary": rs_val, - } - else: - optional_params["reasoning_effort"] = {"summary": rs_val} return responses_api_bridge.completion( # pyright: ignore[reportReturnType] # bridge returns a coroutine on the acompletion path; awaited by the async caller model=model, @@ -6168,17 +6155,10 @@ def completion( api_base=api_base, acompletion=acompletion, logging_obj=logging, - optional_params=optional_params, - litellm_params=( - { - **litellm_params, - "api_base": bedrock_responses_bridge_api_base( - api_base, litellm_params.get("aws_bedrock_runtime_endpoint") - ), - } - if bedrock_bridges_to_responses - else litellm_params + optional_params=with_reasoning_summary_in_effort( + stripped_optional_params, resolved_reasoning_effort, rs_val ), + litellm_params=bridge_litellm_params, timeout=timeout, client=client, # pass AsyncOpenAI, OpenAI client custom_llm_provider=custom_llm_provider, diff --git a/litellm/utils.py b/litellm/utils.py index 618f77efe03..6051e0841f7 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -11011,7 +11011,7 @@ def reasoning_effort_with_summary_alias( def strip_reasoning_summary_aliases_from_optional_params( - optional_params: dict, + optional_params: Mapping[str, object], ) -> tuple[dict, object | None]: """Copy optional_params; remove reasoningSummary aliases from top-level and extra_body.""" op: Final = dict(optional_params) @@ -11035,6 +11035,21 @@ def strip_reasoning_summary_aliases_from_optional_params( return op, rs_val +def with_reasoning_summary_in_effort( + optional_params: Mapping[str, object], resolved_reasoning_effort: object, reasoning_summary: object | None +) -> Mapping[str, object]: + if isinstance(resolved_reasoning_effort, dict) and "summary" in resolved_reasoning_effort: + return {**optional_params, "reasoning_effort": resolved_reasoning_effort} + if reasoning_summary is None: + return optional_params + effort: Final = optional_params.get("reasoning_effort", resolved_reasoning_effort) + if isinstance(effort, dict): + return {**optional_params, "reasoning_effort": {**effort, "summary": reasoning_summary}} + if effort is not None: + return {**optional_params, "reasoning_effort": {"effort": effort, "summary": reasoning_summary}} + return {**optional_params, "reasoning_effort": {"summary": reasoning_summary}} + + def get_non_default_transcription_params(kwargs: Mapping[str, object]) -> dict[str, object]: from litellm.constants import OPENAI_TRANSCRIPTION_PARAMS From bc4ac51c8e8a40a701ea21895bbe9d758172a488 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 02:20:21 -0700 Subject: [PATCH 14/14] style(ui): drop the comment that repeats the Responses API tool call check --- .../src/components/logs/detail/sections/ToolsSection/utils.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts index 789fa4f0f88..25eeb285e75 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts +++ b/ui/litellm-dashboard/src/components/logs/detail/sections/ToolsSection/utils.ts @@ -80,7 +80,6 @@ function extractToolCallsFromResponse(log: LogEntry): ToolCall[] { return responseData.tool_calls; } - // Responses API format: response.output[].type === "function_call" if (Array.isArray(responseData.output)) { const toolCalls = toolCallsFromOutput(responseData.output); if (toolCalls.length > 0) return toolCalls;