diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index 648cceb79b0..8eec07dadda 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -30,8 +30,6 @@ EXTRA_BOOLEAN_KEYS = frozenset( "gemini_audio_only_live", "uses_embed_content", "use_openai_responses_path", - "use_bedrock_runtime_chat_completions", - "bedrock_runtime_chat_completions_tools_require_reasoning_none", "bedrock_converse_supports_strict_tools", "thinking_always_on", } diff --git a/litellm/llms/bedrock/chat/chat_completions/transformation.py b/litellm/llms/bedrock/chat/chat_completions/transformation.py index 49218270064..18ba06fee0b 100644 --- a/litellm/llms/bedrock/chat/chat_completions/transformation.py +++ b/litellm/llms/bedrock/chat/chat_completions/transformation.py @@ -3,7 +3,7 @@ Native OpenAI Chat Completions on Amazon Bedrock Runtime. AWS serves this surface at ``https://bedrock-runtime.{region}.amazonaws.com/openai/v1/chat/completions`` -for the models whose price-map entry sets ``use_bedrock_runtime_chat_completions`` +for the models whose price-map entry sets ``supports_bedrock_runtime_chat_completions`` (Grok 4.6, gpt-oss, the GPT-5.6 family): chat completions stay chat completions instead of being rewritten to Converse. @@ -19,6 +19,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal import httpx +from typing_extensions import assert_never import litellm from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -72,6 +73,8 @@ class ReasoningTagSplitter: return self._feed_start(self.pending + text) case "reasoning": return self._feed_reasoning(self.pending + text) + case _: + assert_never(self.phase) def _feed_start(self, buffered: str) -> tuple["ReasoningTagSplitter", str, str]: if buffered.startswith(REASONING_OPEN_TAG): diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 124ef7c646c..df30eddd856 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -803,22 +803,33 @@ def _bedrock_price_map_flag(model: str, flag: str) -> bool: def uses_bedrock_runtime_chat_completions(model: str) -> bool: """Whether this Bedrock model should use runtime native Chat Completions. - Data-driven from the price-map ``use_bedrock_runtime_chat_completions`` flag + Data-driven from the price-map ``supports_bedrock_runtime_chat_completions`` flag so onboarding a model is a JSON change. Explicit ``converse/`` still wins in ``get_bedrock_route`` because prefix routes are checked first, and a request that needs a Converse-only feature (``bedrock_request_needs_converse``) is served by Converse even on a flagged model. """ - return _bedrock_price_map_flag(model, "use_bedrock_runtime_chat_completions") + return _bedrock_price_map_flag(model, "supports_bedrock_runtime_chat_completions") -def bedrock_runtime_chat_completions_tools_require_reasoning_none(model: str) -> bool: - """Whether AWS's native Chat Completions only serves this model's function tools with ``reasoning_effort="none"``. +def bedrock_runtime_chat_completions_serves_tools_with_reasoning(model: str) -> bool: + """Whether AWS's native Chat Completions serves this model's function tools with any ``reasoning_effort``. - Data-driven from the price-map ``bedrock_runtime_chat_completions_tools_require_reasoning_none`` - flag (the GPT-5.6 family). Converse serves tools with any effort, so those requests fall back to it. + Data-driven from the price-map ``supports_bedrock_runtime_chat_completions_tools_with_reasoning`` + flag (gpt-oss, Grok). Without it AWS only takes tools with ``reasoning_effort="none"`` + (the GPT-5.6 family), and Converse serves tools with any effort, so those requests fall back to it. """ - return _bedrock_price_map_flag(model, "bedrock_runtime_chat_completions_tools_require_reasoning_none") + return _bedrock_price_map_flag(model, "supports_bedrock_runtime_chat_completions_tools_with_reasoning") + + +def bedrock_runtime_chat_completions_enforces_response_format(model: str) -> bool: + """Whether AWS's native Chat Completions enforces a ``response_format`` schema for this model. + + Data-driven from the price-map ``supports_bedrock_runtime_chat_completions_response_format`` flag + (GPT-5.6, Grok). Without it AWS accepts the field and answers with unconstrained text (gpt-oss), so + Converse, which emulates the schema through a forced ``json_tool_call`` tool, serves those requests. + """ + return _bedrock_price_map_flag(model, "supports_bedrock_runtime_chat_completions_response_format") BEDROCK_CONVERSE_ONLY_REQUEST_KEYS: Final = frozenset( @@ -826,26 +837,52 @@ BEDROCK_CONVERSE_ONLY_REQUEST_KEYS: Final = frozenset( ) +def _response_format_constrains_output(response_format: object) -> bool: + if response_format is None: + return False + return not (isinstance(response_format, Mapping) and response_format.get("type") == "text") + + def bedrock_request_needs_converse(model: str, request_params: Mapping[str, object]) -> bool: """Whether a request on a runtime-Chat-Completions model must still be served by Converse. Converse-shaped body keys (``BEDROCK_CONVERSE_ONLY_REQUEST_KEYS``) are rejected as malformed input by AWS's native OpenAI surface, operator-owned request metadata is only written onto the Converse body, - and function tools on a ``bedrock_runtime_chat_completions_tools_require_reasoning_none`` model are - rejected there unless ``reasoning_effort`` is exactly ``"none"``. + function tools on a model without ``supports_bedrock_runtime_chat_completions_tools_with_reasoning`` + are rejected there unless ``reasoning_effort`` is exactly ``"none"``, and a constraining + ``response_format`` on a model without ``supports_bedrock_runtime_chat_completions_response_format`` + is only honored by Converse. """ if any(request_params.get(key) is not None for key in BEDROCK_CONVERSE_ONLY_REQUEST_KEYS): return True if bedrock_request_metadata_is_owned(): return True + if _response_format_constrains_output( + request_params.get("response_format") + ) and not bedrock_runtime_chat_completions_enforces_response_format(model): + return True if not request_params.get("tools"): return False return ( - bedrock_runtime_chat_completions_tools_require_reasoning_none(model) + not bedrock_runtime_chat_completions_serves_tools_with_reasoning(model) and request_params.get("reasoning_effort") != "none" ) +def bedrock_route_for_request( + model: str, request_params: Mapping[str, object], additional_drop_params: Sequence[str] | None +) -> BedrockRoute: + """The route for one request, decided from the caller's raw params before any provider mapping. + + Param mapping and dispatch both call this with the same inputs, so a request that falls back to + Converse is mapped with the Converse config and sent to Converse, never one without the other. + """ + dropped: Final = frozenset(additional_drop_params or ()) + return BedrockModelInfo.get_bedrock_route( + model, {key: value for key, value in request_params.items() if key not in dropped} + ) + + def strip_bedrock_throughput_suffix(model: str) -> str: """Strip throughput tier suffixes and context window suffixes from Bedrock model names.""" import re diff --git a/litellm/main.py b/litellm/main.py index ed78f209ca2..539479fb224 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -104,7 +104,7 @@ from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig from litellm.llms.base_llm.base_model_iterator import ( convert_model_response_to_streaming, ) -from litellm.llms.bedrock.common_utils import BedrockModelInfo +from litellm.llms.bedrock.common_utils import BedrockModelInfo, bedrock_route_for_request from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config @@ -4078,7 +4078,9 @@ def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes if "aws_region_name" not in optional_params or optional_params["aws_region_name"] is None: optional_params["aws_region_name"] = aws_bedrock_client.meta.region_name - bedrock_route: Final = BedrockModelInfo.get_bedrock_route(model, optional_params) + bedrock_route: Final = bedrock_route_for_request( + model, ctx.request_params, ctx.kwargs.get("additional_drop_params") + ) if bedrock_route == "claude_platform": provider_config = ProviderConfigManager.get_provider_chat_config( model=model, @@ -5686,6 +5688,7 @@ def completion( optional_params=optional_params, organization=organization, provider_config=provider_config, + request_params={**optional_param_args, **non_default_params}, shared_session=shared_session, stream=stream, temperature=temperature, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 05658ce2d15..6dbf246fd70 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -40870,7 +40870,8 @@ "output_cost_per_token": 0.0 }, "openai.gpt-oss-120b-1:0": { - "use_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -40884,7 +40885,8 @@ "supports_tool_choice": true }, "openai.gpt-oss-20b-1:0": { - "use_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -47234,7 +47236,9 @@ "input_cost_per_token": 2.64e-06, "output_cost_per_token": 7.92e-06, "cache_read_input_token_cost": 6.6e-07, - "use_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, "max_output_tokens": 500000, @@ -58443,8 +58447,8 @@ "supports_web_search": true }, "us.openai.gpt-5.6-sol": { - "use_bedrock_runtime_chat_completions": true, - "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 4.4e-06, "input_cost_per_token_above_272k_tokens": 8.8e-06, "cache_creation_input_token_cost": 5.5e-06, @@ -58474,8 +58478,8 @@ "supports_vision": true }, "global.openai.gpt-5.6-sol": { - "use_bedrock_runtime_chat_completions": true, - "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "cache_creation_input_token_cost": 5e-06, @@ -58505,8 +58509,8 @@ "supports_vision": true }, "us.openai.gpt-5.6-terra": { - "use_bedrock_runtime_chat_completions": true, - "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 2.2e-06, "input_cost_per_token_above_272k_tokens": 4.4e-06, "cache_creation_input_token_cost": 2.75e-06, @@ -58536,8 +58540,8 @@ "supports_vision": true }, "global.openai.gpt-5.6-terra": { - "use_bedrock_runtime_chat_completions": true, - "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "cache_creation_input_token_cost": 2.5e-06, @@ -58567,8 +58571,8 @@ "supports_vision": true }, "us.openai.gpt-5.6-luna": { - "use_bedrock_runtime_chat_completions": true, - "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, "cache_creation_input_token_cost": 2.75e-07, @@ -58598,8 +58602,8 @@ "supports_vision": true }, "global.openai.gpt-5.6-luna": { - "use_bedrock_runtime_chat_completions": true, - "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 2e-07, "input_cost_per_token_above_272k_tokens": 4e-07, "cache_creation_input_token_cost": 2.5e-07, @@ -58909,7 +58913,9 @@ "input_cost_per_token": 2.2e-06, "output_cost_per_token": 6.6e-06, "cache_read_input_token_cost": 5.5e-07, - "use_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, "max_output_tokens": 500000, @@ -58925,7 +58931,9 @@ "input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, "cache_read_input_token_cost": 5e-07, - "use_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, "max_output_tokens": 500000, diff --git a/litellm/types/completion.py b/litellm/types/completion.py index c1c6cc9ed1c..1e6cfc0ee33 100644 --- a/litellm/types/completion.py +++ b/litellm/types/completion.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Callable, Coroutine, Iterable +from collections.abc import Callable, Coroutine, Iterable, Mapping from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal, Union @@ -229,6 +229,7 @@ class _CompletionDispatchContext: optional_params: dict organization: str | None provider_config: BaseConfig | None + request_params: Mapping[str, object] shared_session: ClientSession | None stream: bool | None temperature: float | None diff --git a/litellm/utils.py b/litellm/utils.py index 82c8069770d..7226b9a865c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -401,7 +401,7 @@ if TYPE_CHECKING: BaseVectorStoreFilesConfig, ) from litellm.llms.base_llm.videos.transformation import BaseVideoConfig - from litellm.llms.bedrock.common_utils import BedrockModelInfo, BedrockRoute + from litellm.llms.bedrock.common_utils import BedrockRoute from litellm.llms.bedrock.embed.amazon_nova_transformation import ( AmazonNovaEmbeddingConfig, ) @@ -3353,12 +3353,9 @@ def _should_drop_param(k, additional_drop_params) -> bool: def _bedrock_route_for_request( model: str, passed_params: Mapping[str, object], additional_drop_params: list | None ) -> BedrockRoute: - from litellm.llms.bedrock.common_utils import BedrockModelInfo + from litellm.llms.bedrock.common_utils import bedrock_route_for_request - return BedrockModelInfo.get_bedrock_route( - model, - {k: v for k, v in passed_params.items() if not _should_drop_param(k, additional_drop_params)}, - ) + return bedrock_route_for_request(model, passed_params, additional_drop_params) def _get_non_default_params(passed_params: dict, default_params: dict, additional_drop_params: list | None) -> dict: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 05658ce2d15..6dbf246fd70 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -40870,7 +40870,8 @@ "output_cost_per_token": 0.0 }, "openai.gpt-oss-120b-1:0": { - "use_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -40884,7 +40885,8 @@ "supports_tool_choice": true }, "openai.gpt-oss-20b-1:0": { - "use_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -47234,7 +47236,9 @@ "input_cost_per_token": 2.64e-06, "output_cost_per_token": 7.92e-06, "cache_read_input_token_cost": 6.6e-07, - "use_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, "max_output_tokens": 500000, @@ -58443,8 +58447,8 @@ "supports_web_search": true }, "us.openai.gpt-5.6-sol": { - "use_bedrock_runtime_chat_completions": true, - "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 4.4e-06, "input_cost_per_token_above_272k_tokens": 8.8e-06, "cache_creation_input_token_cost": 5.5e-06, @@ -58474,8 +58478,8 @@ "supports_vision": true }, "global.openai.gpt-5.6-sol": { - "use_bedrock_runtime_chat_completions": true, - "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "cache_creation_input_token_cost": 5e-06, @@ -58505,8 +58509,8 @@ "supports_vision": true }, "us.openai.gpt-5.6-terra": { - "use_bedrock_runtime_chat_completions": true, - "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 2.2e-06, "input_cost_per_token_above_272k_tokens": 4.4e-06, "cache_creation_input_token_cost": 2.75e-06, @@ -58536,8 +58540,8 @@ "supports_vision": true }, "global.openai.gpt-5.6-terra": { - "use_bedrock_runtime_chat_completions": true, - "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "cache_creation_input_token_cost": 2.5e-06, @@ -58567,8 +58571,8 @@ "supports_vision": true }, "us.openai.gpt-5.6-luna": { - "use_bedrock_runtime_chat_completions": true, - "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, "cache_creation_input_token_cost": 2.75e-07, @@ -58598,8 +58602,8 @@ "supports_vision": true }, "global.openai.gpt-5.6-luna": { - "use_bedrock_runtime_chat_completions": true, - "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 2e-07, "input_cost_per_token_above_272k_tokens": 4e-07, "cache_creation_input_token_cost": 2.5e-07, @@ -58909,7 +58913,9 @@ "input_cost_per_token": 2.2e-06, "output_cost_per_token": 6.6e-06, "cache_read_input_token_cost": 5.5e-07, - "use_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, "max_output_tokens": 500000, @@ -58925,7 +58931,9 @@ "input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, "cache_read_input_token_cost": 5e-07, - "use_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions": true, + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, "max_output_tokens": 500000, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 9db1c8eedc4..1a3523cdff2 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -74,9 +74,6 @@ "xhigh" ] }, - "bedrock_runtime_chat_completions_tools_require_reasoning_none": { - "type": "boolean" - }, "cache_creation_input_audio_token_cost": { "type": "number", "minimum": 0 @@ -830,6 +827,15 @@ "supports_audio_output": { "type": "boolean" }, + "supports_bedrock_runtime_chat_completions": { + "type": "boolean" + }, + "supports_bedrock_runtime_chat_completions_response_format": { + "type": "boolean" + }, + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": { + "type": "boolean" + }, "supports_computer_use": { "type": "boolean" }, @@ -1000,9 +1006,6 @@ "minimum": 0, "description": "Provider default tokens-per-minute limit." }, - "use_bedrock_runtime_chat_completions": { - "type": "boolean" - }, "use_openai_responses_path": { "type": "boolean" }, diff --git a/tests/test_litellm/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/test_litellm/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 230e323ace3..ce2e5b7e8f7 100644 --- a/tests/test_litellm/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -4,6 +4,7 @@ import json import httpx import pytest +from pydantic import BaseModel import litellm from litellm.llms.bedrock.chat.chat_completions.transformation import ( @@ -17,6 +18,7 @@ from litellm.llms.bedrock.common_utils import ( BEDROCK_CONVERSE_ONLY_REQUEST_KEYS, BedrockModelInfo, bedrock_request_needs_converse, + bedrock_route_for_request, get_bedrock_chat_config, uses_bedrock_runtime_chat_completions, ) @@ -471,7 +473,7 @@ def test_converse_only_request_keys_go_to_converse(local_cost_map, fake_aws_env, ) assert requests[0].url.raw_path.endswith(b"/model/openai.gpt-oss-20b-1%3A0/converse") - (key, value), = converse_only_param.items() + ((key, value),) = converse_only_param.items() assert json.loads(requests[0].content)[key] == value @@ -620,3 +622,124 @@ def test_streaming_handler_keeps_native_reasoning_next_to_the_tagged_split(): assert parsed.choices[0].delta.reasoning_content == "native tagged" assert parsed.choices[0].delta.content == "Hi" + + +RESPONSE_FORMAT_JSON_SCHEMA = { + "type": "json_schema", + "json_schema": { + "name": "answer", + "schema": {"type": "object", "properties": {"word": {"type": "string"}}, "required": ["word"]}, + "strict": True, + }, +} + + +class Answer(BaseModel): + word: str + + +@pytest.mark.parametrize("model", ["openai.gpt-oss-20b-1:0", "bedrock/openai.gpt-oss-120b-1:0"]) +@pytest.mark.parametrize( + "response_format, expected_route", + [ + (RESPONSE_FORMAT_JSON_SCHEMA, "converse"), + ({"type": "json_object"}, "converse"), + (Answer, "converse"), + ({"type": "text"}, "chat_completions"), + (None, "chat_completions"), + ], + ids=["json_schema", "json_object", "pydantic", "text", "none"], +) +def test_gpt_oss_response_format_falls_back_to_converse(local_cost_map, model, response_format, expected_route): + params = {"response_format": response_format} + assert bedrock_request_needs_converse(model, params) is (expected_route == "converse") + assert BedrockModelInfo.get_bedrock_route(model, params) == expected_route + + +@pytest.mark.parametrize("model", ["global.openai.gpt-5.6-sol", "us.xai.grok-4.6", "bedrock/us-gov.xai.grok-4.6"]) +def test_response_format_stays_on_chat_completions_where_aws_enforces_it(local_cost_map, model): + params = {"response_format": RESPONSE_FORMAT_JSON_SCHEMA} + assert bedrock_request_needs_converse(model, params) is False + assert BedrockModelInfo.get_bedrock_route(model, params) == "chat_completions" + + +SYNTHETIC_NATIVE_MODEL = "vendor.native-model-v1:0" + + +@pytest.mark.parametrize( + "capability_flags, request_params, needs_converse", + [ + ({}, {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}, True), + ({}, {"tools": [GET_WEATHER_TOOL]}, True), + ({}, {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "none"}, False), + ( + {"supports_bedrock_runtime_chat_completions_tools_with_reasoning": True}, + {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}, + False, + ), + ({}, {"response_format": RESPONSE_FORMAT_JSON_SCHEMA}, True), + ( + {"supports_bedrock_runtime_chat_completions_response_format": True}, + {"response_format": RESPONSE_FORMAT_JSON_SCHEMA}, + False, + ), + ( + {"supports_bedrock_runtime_chat_completions_response_format": True}, + {"response_format": RESPONSE_FORMAT_JSON_SCHEMA, "tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}, + True, + ), + ], +) +def test_capability_flags_are_read_from_the_cost_map(monkeypatch, capability_flags, request_params, needs_converse): + entry = { + "litellm_provider": "bedrock_converse", + "supports_bedrock_runtime_chat_completions": True, + **capability_flags, + } + monkeypatch.setattr(litellm, "model_cost", {SYNTHETIC_NATIVE_MODEL: entry}) + assert bedrock_request_needs_converse(SYNTHETIC_NATIVE_MODEL, request_params) is needs_converse + route = bedrock_route_for_request(SYNTHETIC_NATIVE_MODEL, request_params, None) + assert (route == "chat_completions") is (not needs_converse) + + +def test_route_for_request_ignores_dropped_params(local_cost_map): + params = {"response_format": RESPONSE_FORMAT_JSON_SCHEMA, "guardrailConfig": {"guardrailIdentifier": "gr-1"}} + assert bedrock_route_for_request("openai.gpt-oss-20b-1:0", params, None) == "converse" + assert bedrock_route_for_request("openai.gpt-oss-20b-1:0", params, ["guardrailConfig"]) == "converse" + assert ( + bedrock_route_for_request("openai.gpt-oss-20b-1:0", params, ["guardrailConfig", "response_format"]) + == "chat_completions" + ) + + +def test_gpt_oss_response_format_goes_to_converse_with_json_tool_call(local_cost_map, fake_aws_env): + requests, client = _recording_client(json=CONVERSE_JSON) + litellm.completion( + model="bedrock/openai.gpt-oss-20b-1:0", + messages=[{"role": "user", "content": "Reply with the single word pong."}], + response_format=RESPONSE_FORMAT_JSON_SCHEMA, + max_tokens=64, + client=client, + ) + + assert requests[0].url.raw_path.endswith(b"/model/openai.gpt-oss-20b-1%3A0/converse") + body = json.loads(requests[0].content) + assert body["toolConfig"]["tools"][0]["toolSpec"]["name"] == "json_tool_call" + assert body["toolConfig"]["toolChoice"] == {"tool": {"name": "json_tool_call"}} + assert body["inferenceConfig"]["maxTokens"] == 64 + assert "response_format" not in body + assert "max_completion_tokens" not in body + + +def test_gpt56_response_format_is_sent_as_is_on_chat_completions(local_cost_map, fake_aws_env): + requests, client = _recording_client(json=_chat_completion_json('{"word": "pong"}', "global.openai.gpt-5.6-sol")) + response = litellm.completion( + model="bedrock/global.openai.gpt-5.6-sol", + messages=[{"role": "user", "content": "Reply with the single word pong."}], + response_format=RESPONSE_FORMAT_JSON_SCHEMA, + client=client, + ) + + assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions" + assert json.loads(requests[0].content)["response_format"] == RESPONSE_FORMAT_JSON_SCHEMA + assert response.choices[0].message.content == '{"word": "pong"}' diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 963bef1114a..d049d83c6a9 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -878,8 +878,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_video_input": {"type": "boolean"}, "supports_vision": {"type": "boolean"}, "supports_web_search": {"type": "boolean"}, - "use_bedrock_runtime_chat_completions": {"type": "boolean"}, - "bedrock_runtime_chat_completions_tools_require_reasoning_none": {"type": "boolean"}, + "supports_bedrock_runtime_chat_completions": {"type": "boolean"}, + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": {"type": "boolean"}, + "supports_bedrock_runtime_chat_completions_response_format": {"type": "boolean"}, "supports_url_context": {"type": "boolean"}, "supports_multimodal": {"type": "boolean"}, "uses_embed_content": {"type": "boolean"}, diff --git a/tests/test_litellm/types/test_completion.py b/tests/test_litellm/types/test_completion.py index cd51913c5dd..482dd351ad1 100644 --- a/tests/test_litellm/types/test_completion.py +++ b/tests/test_litellm/types/test_completion.py @@ -181,6 +181,7 @@ def _build_dispatch_context() -> _CompletionDispatchContext: optional_params={}, organization=None, provider_config=None, + request_params={}, shared_session=None, stream=None, temperature=None,