diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index 5a2a56a5b21..648cceb79b0 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -31,6 +31,7 @@ EXTRA_BOOLEAN_KEYS = frozenset( "uses_embed_content", "use_openai_responses_path", "use_bedrock_runtime_chat_completions", + "bedrock_runtime_chat_completions_tools_require_reasoning_none", "bedrock_converse_supports_strict_tools", "thinking_always_on", } diff --git a/litellm/llms/bedrock/chat/chat_completions/transformation.py b/litellm/llms/bedrock/chat/chat_completions/transformation.py index 3bf6b2a2ffd..49218270064 100644 --- a/litellm/llms/bedrock/chat/chat_completions/transformation.py +++ b/litellm/llms/bedrock/chat/chat_completions/transformation.py @@ -2,30 +2,170 @@ Native OpenAI Chat Completions on Amazon Bedrock Runtime. AWS serves this surface at -``https://bedrock-runtime.{region}.amazonaws.com/openai/v1/chat/completions``. -Grok 4.6 on runtime is one of the models that uses it: chat completions stay -chat completions instead of being rewritten to Converse. +``https://bedrock-runtime.{region}.amazonaws.com/openai/v1/chat/completions`` +for the models whose price-map entry sets ``use_bedrock_runtime_chat_completions`` +(Grok 4.6, gpt-oss, the GPT-5.6 family): chat completions stay chat completions +instead of being rewritten to Converse. -Usage: model="us.xai.grok-4.6" or model="bedrock/us.xai.grok-4.6" -Explicit ``bedrock/converse/...`` still uses Converse. +Usage: model="us.xai.grok-4.6", model="bedrock/openai.gpt-oss-20b-1:0" or +model="bedrock/global.openai.gpt-5.6-sol". Explicit ``bedrock/converse/...`` +still uses Converse, and so does a request that needs a Converse-only feature +(``bedrock_request_needs_converse`` in ``common_utils``). """ -from collections.abc import AsyncIterator, Iterator -from typing import Any, Final +from collections.abc import AsyncIterator, Iterator, Mapping +from dataclasses import dataclass, replace +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal import httpx import litellm -from litellm._logging import verbose_logger from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import BedrockError, strip_bedrock_routing_prefix +from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import Choices, ModelResponse, ModelResponseStream + +if TYPE_CHECKING: + import tiktoken + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +REASONING_OPEN_TAG: Final = "" +REASONING_CLOSE_TAG: Final = "" + + +def _held_close_tag_prefix(text: str) -> int: + return next( + ( + size + for size in range(min(len(text), len(REASONING_CLOSE_TAG) - 1), 0, -1) + if REASONING_CLOSE_TAG.startswith(text[-size:]) + ), + 0, + ) + + +@dataclass(frozen=True, slots=True) +class ReasoningTagSplitter: + """ + The same split for a stream of content deltas, where a tag can arrive across chunks. + + ``feed`` returns the next state plus the reasoning and content text the delta contributes; + ``flush`` releases what the stream ended on before a tag resolved. + """ + + phase: Literal["start", "reasoning", "after_close", "content"] = "start" + pending: str = "" + + def feed(self, text: str) -> tuple["ReasoningTagSplitter", str, str]: + match self.phase: + case "content": + return self, "", text + case "after_close": + content: Final = text.lstrip() + return (replace(self, phase="content") if content else self), "", content + case "start": + return self._feed_start(self.pending + text) + case "reasoning": + return self._feed_reasoning(self.pending + text) + + def _feed_start(self, buffered: str) -> tuple["ReasoningTagSplitter", str, str]: + if buffered.startswith(REASONING_OPEN_TAG): + return replace(self, phase="reasoning", pending="")._feed_reasoning(buffered[len(REASONING_OPEN_TAG) :]) + if REASONING_OPEN_TAG.startswith(buffered): + return replace(self, pending=buffered), "", "" + return replace(self, phase="content", pending=""), "", buffered + + def _feed_reasoning(self, buffered: str) -> tuple["ReasoningTagSplitter", str, str]: + close_at: Final = buffered.find(REASONING_CLOSE_TAG) + if close_at >= 0: + after_close: Final = replace(self, phase="after_close", pending="") + next_state, _, content = after_close.feed(buffered[close_at + len(REASONING_CLOSE_TAG) :]) + return next_state, buffered[:close_at], content + held: Final = _held_close_tag_prefix(buffered) + return replace(self, pending=buffered[len(buffered) - held :]), buffered[: len(buffered) - held], "" + + def flush(self) -> tuple["ReasoningTagSplitter", str, str]: + drained: Final = replace(self, phase="content", pending="") + if self.phase == "reasoning": + return drained, self.pending, "" + return drained, "", self.pending + + +def _split_streamed_content( + splitter: ReasoningTagSplitter, content: str | None, finished: bool +) -> tuple[ReasoningTagSplitter, str, str]: + fed_state, fed_reasoning, fed_content = splitter.feed(content or "") + if not finished: + return fed_state, fed_reasoning, fed_content + drained, flushed_reasoning, flushed_content = fed_state.flush() + return drained, fed_reasoning + flushed_reasoning, fed_content + flushed_content + + +def split_reasoning_tag(content: str) -> tuple[str | None, str]: + """ + Split gpt-oss's inline ``...`` prefix out of a complete message. + + Runs the streaming splitter over the whole message, so a streamed and a non-streamed + response to the same completion split identically. Returns ``(None, content)`` when the + message does not start with the tag. + """ + _, reasoning, body = _split_streamed_content(ReasoningTagSplitter(), content, finished=True) + return reasoning or None, body + + +class BedrockRuntimeChatCompletionsStreamingHandler(OpenAIChatCompletionStreamingHandler): + """OpenAI chunk parsing plus the ```` split, tracked per choice index.""" + + def __init__( + self, + streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, + sync_stream: bool, + json_mode: bool | None = False, + ) -> None: + super().__init__(streaming_response=streaming_response, sync_stream=sync_stream, json_mode=json_mode) + self._splitters: Mapping[int, ReasoningTagSplitter] = MappingProxyType({}) + + def chunk_parser(self, chunk: dict) -> ModelResponseStream: # mutable-ok: BaseModelResponseIterator signature + parsed: Final = super().chunk_parser(chunk) + for choice in parsed.choices: + next_state, reasoning, content = _split_streamed_content( + self._splitters.get(choice.index, ReasoningTagSplitter()), + choice.delta.content, + choice.finish_reason is not None, + ) + self._splitters = MappingProxyType({**self._splitters, choice.index: next_state}) + if reasoning: + choice.delta.reasoning_content = f"{getattr(choice.delta, 'reasoning_content', None) or ''}{reasoning}" + if content or choice.delta.content is not None: + choice.delta.content = content + return parsed + + +def with_max_completion_tokens(params: Mapping[str, object]) -> Mapping[str, object]: + """ + Send the caller's ``max_tokens`` as ``max_completion_tokens``. + + Every model on this surface accepts ``max_completion_tokens`` and the GPT-5.6 family + rejects ``max_tokens``; an explicit ``max_completion_tokens`` wins when both are set. + """ + if "max_tokens" not in params: + return params + return MappingProxyType( + { + key: value + for key, value in (("max_completion_tokens", params["max_tokens"]), *params.items()) + if key != "max_tokens" + } + ) class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): - def __init__(self, aws_signer: BaseAWSLLM | None = None): + def __init__(self, aws_signer: BaseAWSLLM | None = None) -> None: super().__init__() self._aws_signer: Final = aws_signer or BaseAWSLLM() @@ -34,7 +174,10 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): return "bedrock" def get_error_class( - self, error_message: str, status_code: int, headers: dict[str, object] | httpx.Headers + self, + error_message: str, + status_code: int, + headers: dict[str, object] | httpx.Headers, # mutable-ok: BaseConfig signature ) -> BaseLLMException: return BedrockError(status_code=status_code, message=error_message, headers=headers) @@ -43,13 +186,15 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): api_base: str | None, api_key: str | None, model: str, - optional_params: dict, - litellm_params: dict, + optional_params: dict, # mutable-ok: BaseConfig signature + litellm_params: dict, # mutable-ok: BaseConfig signature stream: bool | None = None, ) -> str: if api_base is not None and "chat/completions" in api_base: return api_base.rstrip("/") - aws_region_name: Final = self._aws_signer._get_aws_region_name(optional_params=optional_params, model=model) + aws_region_name: Final = self._aws_signer._get_aws_region_name( # pyright: ignore[reportPrivateUsage] # BaseAWSLLM has no public region resolver + optional_params=optional_params, model=model + ) endpoint_url, _ = self._aws_signer.get_runtime_endpoint( api_base=api_base, aws_bedrock_runtime_endpoint=optional_params.get("aws_bedrock_runtime_endpoint"), @@ -64,16 +209,16 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): def sign_request( self, - headers: dict, - optional_params: dict, - request_data: dict, + headers: dict, # mutable-ok: BaseConfig signature + optional_params: dict, # mutable-ok: BaseConfig signature + request_data: dict, # mutable-ok: BaseConfig signature api_base: str, api_key: str | None = None, model: str | None = None, stream: bool | None = None, fake_stream: bool | None = None, - ) -> tuple[dict, bytes | None]: - return self._aws_signer._sign_request( + ) -> tuple[dict, bytes | None]: # mutable-ok: BaseConfig signature + return self._aws_signer._sign_request( # pyright: ignore[reportPrivateUsage] # BaseAWSLLM has no public signer service_name="bedrock", headers=headers, optional_params=optional_params, @@ -85,21 +230,44 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): fake_stream=fake_stream, ) + def map_openai_params( + self, + non_default_params: dict, # mutable-ok: BaseConfig signature + optional_params: dict, # mutable-ok: BaseConfig signature + model: str, + drop_params: bool, + replace_max_completion_tokens_with_max_tokens: bool = False, + ) -> dict: # mutable-ok: BaseConfig signature + mapped: Final = super().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=drop_params, + replace_max_completion_tokens_with_max_tokens=replace_max_completion_tokens_with_max_tokens, + ) + return dict(with_max_completion_tokens(mapped)) # mutable-ok: get_optional_params keeps filling this dict + + def _inference_params( + self, optional_params: Mapping[str, object] + ) -> dict[str, object]: # mutable-ok: BaseConfig signature of transform_request + return { # mutable-ok: OpenAILikeChatConfig.transform_request takes a plain dict + key: value + for key, value in optional_params.items() + if key not in self._aws_signer.aws_authentication_params + } + def transform_request( self, model: str, - messages: list[AllMessageValues], - optional_params: dict, - litellm_params: dict, - headers: dict, - ) -> dict: - inference_params: Final = { - k: v for k, v in optional_params.items() if k not in self._aws_signer.aws_authentication_params - } + messages: list[AllMessageValues], # mutable-ok: BaseConfig signature + optional_params: dict, # mutable-ok: BaseConfig signature + litellm_params: dict, # mutable-ok: BaseConfig signature + headers: dict, # mutable-ok: BaseConfig signature + ) -> dict: # mutable-ok: BaseConfig signature return super().transform_request( model=strip_bedrock_routing_prefix(model), messages=messages, - optional_params=inference_params, + optional_params=self._inference_params(optional_params), litellm_params=litellm_params, headers=headers, ) @@ -107,33 +275,68 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): async def async_transform_request( self, model: str, - messages: list[AllMessageValues], - optional_params: dict, - litellm_params: dict, - headers: dict, - ) -> dict: - inference_params: Final = { - k: v for k, v in optional_params.items() if k not in self._aws_signer.aws_authentication_params - } + messages: list[AllMessageValues], # mutable-ok: BaseConfig signature + optional_params: dict, # mutable-ok: BaseConfig signature + litellm_params: dict, # mutable-ok: BaseConfig signature + headers: dict, # mutable-ok: BaseConfig signature + ) -> dict: # mutable-ok: BaseConfig signature return await super().async_transform_request( model=strip_bedrock_routing_prefix(model), messages=messages, - optional_params=inference_params, + optional_params=self._inference_params(optional_params), litellm_params=litellm_params, headers=headers, ) + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: "LiteLLMLoggingObj", + request_data: dict, # mutable-ok: BaseConfig signature + messages: list[AllMessageValues], # mutable-ok: BaseConfig signature + optional_params: dict, # mutable-ok: BaseConfig signature + litellm_params: dict, # mutable-ok: BaseConfig signature + encoding: "tiktoken.Encoding | None", + api_key: str | None = None, + json_mode: bool | None = None, + ) -> ModelResponse: + response: Final = super().transform_response( + model=model, + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data=request_data, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + encoding=encoding, + api_key=api_key, + json_mode=json_mode, + ) + for choice in response.choices: + if not isinstance(choice, Choices) or not isinstance(choice.message.content, str): + continue + reasoning, content = split_reasoning_tag(choice.message.content) + if reasoning is not None: + choice.message.reasoning_content = ( + f"{getattr(choice.message, 'reasoning_content', None) or ''}{reasoning}" + ) + choice.message.content = content + return response + def validate_environment( self, - headers: dict, + headers: dict, # mutable-ok: BaseConfig signature model: str, - messages: list[AllMessageValues], - optional_params: dict, - litellm_params: dict, + messages: list[AllMessageValues], # mutable-ok: BaseConfig signature + optional_params: dict, # mutable-ok: BaseConfig signature + litellm_params: dict, # mutable-ok: BaseConfig signature api_key: str | None = None, api_base: str | None = None, - ) -> dict: - headers = super().validate_environment( + ) -> dict: # mutable-ok: BaseConfig signature + validated: Final = super().validate_environment( headers=headers, model=model, messages=messages, @@ -143,31 +346,25 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): api_base=api_base, ) project_id: Final = litellm_params.get("aws_bedrock_project_id") - if project_id: - headers["OpenAI-Project"] = project_id - return headers + if not project_id: + return validated + return {**validated, "OpenAI-Project": project_id} # mutable-ok: BaseConfig signature returns a dict - def get_supported_openai_params(self, model: str) -> list: - base_params: Final = super().get_supported_openai_params(model) - try: - if litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider): - if "reasoning_effort" not in base_params: - base_params.append("reasoning_effort") - except Exception as e: - verbose_logger.debug("AmazonBedrockRuntimeChatCompletionsConfig: error checking reasoning support: %s", e) - return base_params + def get_supported_openai_params(self, model: str) -> list: # mutable-ok: BaseConfig signature + base_params: Final = [param for param in super().get_supported_openai_params(model) if param != "n"] + if "reasoning_effort" in base_params or not litellm.supports_reasoning( + model=model, custom_llm_provider=self.custom_llm_provider + ): + return base_params + return [*base_params, "reasoning_effort"] # mutable-ok: BaseConfig signature returns a list def get_model_response_iterator( self, - streaming_response: Iterator[str] | AsyncIterator[str] | Any, + streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, sync_stream: bool, json_mode: bool | None = False, - ) -> Any: - from litellm.llms.openai.chat.gpt_transformation import ( - OpenAIChatCompletionStreamingHandler, - ) - - return OpenAIChatCompletionStreamingHandler( + ) -> BedrockRuntimeChatCompletionsStreamingHandler: + return BedrockRuntimeChatCompletionsStreamingHandler( streaming_response=streaming_response, sync_stream=sync_stream, json_mode=json_mode, diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index f6384aa97f1..124ef7c646c 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -28,6 +28,7 @@ from litellm.llms.base_llm.anthropic_messages.transformation import ( ) from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.bedrock.request_metadata import bedrock_request_metadata_is_owned from litellm.secret_managers.main import get_secret, get_secret_str from litellm.types.llms.bedrock import AWS_AUTH_PARAM_KEYS, AwsAuthParams @@ -37,6 +38,18 @@ if TYPE_CHECKING: _ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs" _OPENAI_FAMILY_MODEL_RE: Final = re.compile(r"(^|[./])openai\.") +BedrockRoute = Literal[ + "converse", + "invoke", + "claude_platform", + "converse_like", + "agent", + "agentcore", + "async_invoke", + "openai", + "mantle", + "chat_completions", +] def error_response_text(response: httpx.Response) -> str: @@ -782,17 +795,54 @@ def strip_bedrock_routing_prefix(model: str) -> str: return model +def _bedrock_price_map_flag(model: str, flag: str) -> bool: + entries: Final = (litellm.model_cost.get(key) for key in (model, strip_bedrock_routing_prefix(model))) + return any(entry is not None and entry.get(flag) is True for entry in entries) + + def uses_bedrock_runtime_chat_completions(model: str) -> bool: """Whether this Bedrock model should use runtime native Chat Completions. Data-driven from the price-map ``use_bedrock_runtime_chat_completions`` flag so onboarding a model is a JSON change. Explicit ``converse/`` still wins in - ``get_bedrock_route`` because prefix routes are checked first. + ``get_bedrock_route`` because prefix routes are checked first, and a request + that needs a Converse-only feature (``bedrock_request_needs_converse``) is + served by Converse even on a flagged model. """ - stripped: Final = strip_bedrock_routing_prefix(model) - return any( - (litellm.model_cost.get(key) or {}).get("use_bedrock_runtime_chat_completions") is True - for key in (model, stripped) + return _bedrock_price_map_flag(model, "use_bedrock_runtime_chat_completions") + + +def bedrock_runtime_chat_completions_tools_require_reasoning_none(model: str) -> bool: + """Whether AWS's native Chat Completions only serves this model's function tools with ``reasoning_effort="none"``. + + Data-driven from the price-map ``bedrock_runtime_chat_completions_tools_require_reasoning_none`` + flag (the GPT-5.6 family). Converse serves tools with any effort, so those requests fall back to it. + """ + return _bedrock_price_map_flag(model, "bedrock_runtime_chat_completions_tools_require_reasoning_none") + + +BEDROCK_CONVERSE_ONLY_REQUEST_KEYS: Final = frozenset( + ("guardrailConfig", "performanceConfig", "serviceTier", "requestMetadata", "outputConfig") +) + + +def bedrock_request_needs_converse(model: str, request_params: Mapping[str, object]) -> bool: + """Whether a request on a runtime-Chat-Completions model must still be served by Converse. + + Converse-shaped body keys (``BEDROCK_CONVERSE_ONLY_REQUEST_KEYS``) are rejected as malformed input by + AWS's native OpenAI surface, operator-owned request metadata is only written onto the Converse body, + and function tools on a ``bedrock_runtime_chat_completions_tools_require_reasoning_none`` model are + rejected there unless ``reasoning_effort`` is exactly ``"none"``. + """ + if any(request_params.get(key) is not None for key in BEDROCK_CONVERSE_ONLY_REQUEST_KEYS): + return True + if bedrock_request_metadata_is_owned(): + return True + if not request_params.get("tools"): + return False + return ( + bedrock_runtime_chat_completions_tools_require_reasoning_none(model) + and request_params.get("reasoning_effort") != "none" ) @@ -1130,20 +1180,13 @@ class BedrockModelInfo(BaseLLMModelInfo): @staticmethod def get_bedrock_route( model: str, - ) -> Literal[ - "converse", - "invoke", - "claude_platform", - "converse_like", - "agent", - "agentcore", - "async_invoke", - "openai", - "mantle", - "chat_completions", - ]: + request_params: Mapping[str, object] | None = None, + ) -> BedrockRoute: """ Get the bedrock route for the given model. + + ``request_params`` (the caller's chat params) lets a runtime Chat Completions + model fall back to Converse for the requests only Converse can serve. """ route_mappings: dict[ str, @@ -1187,7 +1230,9 @@ class BedrockModelInfo(BaseLLMModelInfo): if is_bedrock_application_inference_profile_arn(model): return "converse" - if uses_bedrock_runtime_chat_completions(model): + if uses_bedrock_runtime_chat_completions(model) and not ( + request_params is not None and bedrock_request_needs_converse(model, request_params) + ): return "chat_completions" base_model: Final = BedrockModelInfo.get_base_model(model) diff --git a/litellm/main.py b/litellm/main.py index 34410f9497c..ed78f209ca2 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4078,7 +4078,7 @@ def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes if "aws_region_name" not in optional_params or optional_params["aws_region_name"] is None: optional_params["aws_region_name"] = aws_bedrock_client.meta.region_name - bedrock_route: Final = BedrockModelInfo.get_bedrock_route(model) + bedrock_route: Final = BedrockModelInfo.get_bedrock_route(model, optional_params) if bedrock_route == "claude_platform": provider_config = ProviderConfigManager.get_provider_chat_config( model=model, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 70314a2e823..05658ce2d15 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -40870,6 +40870,7 @@ "output_cost_per_token": 0.0 }, "openai.gpt-oss-120b-1:0": { + "use_bedrock_runtime_chat_completions": true, "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -40883,6 +40884,7 @@ "supports_tool_choice": true }, "openai.gpt-oss-20b-1:0": { + "use_bedrock_runtime_chat_completions": true, "input_cost_per_token": 7e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -58441,6 +58443,8 @@ "supports_web_search": true }, "us.openai.gpt-5.6-sol": { + "use_bedrock_runtime_chat_completions": true, + "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, "input_cost_per_token": 4.4e-06, "input_cost_per_token_above_272k_tokens": 8.8e-06, "cache_creation_input_token_cost": 5.5e-06, @@ -58470,6 +58474,8 @@ "supports_vision": true }, "global.openai.gpt-5.6-sol": { + "use_bedrock_runtime_chat_completions": true, + "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "cache_creation_input_token_cost": 5e-06, @@ -58499,6 +58505,8 @@ "supports_vision": true }, "us.openai.gpt-5.6-terra": { + "use_bedrock_runtime_chat_completions": true, + "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, "input_cost_per_token": 2.2e-06, "input_cost_per_token_above_272k_tokens": 4.4e-06, "cache_creation_input_token_cost": 2.75e-06, @@ -58528,6 +58536,8 @@ "supports_vision": true }, "global.openai.gpt-5.6-terra": { + "use_bedrock_runtime_chat_completions": true, + "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "cache_creation_input_token_cost": 2.5e-06, @@ -58557,6 +58567,8 @@ "supports_vision": true }, "us.openai.gpt-5.6-luna": { + "use_bedrock_runtime_chat_completions": true, + "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, "cache_creation_input_token_cost": 2.75e-07, @@ -58586,6 +58598,8 @@ "supports_vision": true }, "global.openai.gpt-5.6-luna": { + "use_bedrock_runtime_chat_completions": true, + "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, "input_cost_per_token": 2e-07, "input_cost_per_token_above_272k_tokens": 4e-07, "cache_creation_input_token_cost": 2.5e-07, diff --git a/litellm/utils.py b/litellm/utils.py index b724313641f..82c8069770d 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -401,7 +401,7 @@ if TYPE_CHECKING: BaseVectorStoreFilesConfig, ) from litellm.llms.base_llm.videos.transformation import BaseVideoConfig - from litellm.llms.bedrock.common_utils import BedrockModelInfo + from litellm.llms.bedrock.common_utils import BedrockModelInfo, BedrockRoute from litellm.llms.bedrock.embed.amazon_nova_transformation import ( AmazonNovaEmbeddingConfig, ) @@ -3350,6 +3350,17 @@ def _should_drop_param(k, additional_drop_params) -> bool: return False +def _bedrock_route_for_request( + model: str, passed_params: Mapping[str, object], additional_drop_params: list | None +) -> BedrockRoute: + from litellm.llms.bedrock.common_utils import BedrockModelInfo + + return BedrockModelInfo.get_bedrock_route( + model, + {k: v for k, v in passed_params.items() if not _should_drop_param(k, additional_drop_params)}, + ) + + def _get_non_default_params(passed_params: dict, default_params: dict, additional_drop_params: list | None) -> dict: non_default_params: Final = {} for k, v in passed_params.items(): @@ -4401,9 +4412,17 @@ def get_optional_params( message=f"{custom_llm_provider} does not support parameters: {list(unsupported_params.keys())}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n. \n If you want to use these params dynamically send allowed_openai_params={list(unsupported_params.keys())} in your request.", ) + bedrock_route: Final = ( + _bedrock_route_for_request(model, passed_params, additional_drop_params) + if custom_llm_provider == "bedrock" + else None + ) get_supported_openai_params: Final = getattr(sys.modules[__name__], "get_supported_openai_params") - supported_params = get_supported_openai_params( - model=model, custom_llm_provider=custom_llm_provider, base_model=base_model + supported_params = ( + litellm.AmazonConverseConfig().get_supported_openai_params(model=model) + if bedrock_route == "converse" + and isinstance(provider_config, litellm.AmazonBedrockRuntimeChatCompletionsConfig) + else get_supported_openai_params(model=model, custom_llm_provider=custom_llm_provider, base_model=base_model) ) if supported_params is None: supported_params = get_supported_openai_params(model=model, custom_llm_provider="openai") @@ -4573,7 +4592,6 @@ def get_optional_params( ) elif custom_llm_provider == "bedrock": BedrockModelInfo: Final = getattr(sys.modules[__name__], "BedrockModelInfo") - bedrock_route: Final = BedrockModelInfo.get_bedrock_route(model) bedrock_base_model: Final = BedrockModelInfo.get_base_model(model) if bedrock_route == "converse" or bedrock_route == "converse_like": optional_params = litellm.AmazonConverseConfig().map_openai_params( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 70314a2e823..05658ce2d15 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -40870,6 +40870,7 @@ "output_cost_per_token": 0.0 }, "openai.gpt-oss-120b-1:0": { + "use_bedrock_runtime_chat_completions": true, "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -40883,6 +40884,7 @@ "supports_tool_choice": true }, "openai.gpt-oss-20b-1:0": { + "use_bedrock_runtime_chat_completions": true, "input_cost_per_token": 7e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -58441,6 +58443,8 @@ "supports_web_search": true }, "us.openai.gpt-5.6-sol": { + "use_bedrock_runtime_chat_completions": true, + "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, "input_cost_per_token": 4.4e-06, "input_cost_per_token_above_272k_tokens": 8.8e-06, "cache_creation_input_token_cost": 5.5e-06, @@ -58470,6 +58474,8 @@ "supports_vision": true }, "global.openai.gpt-5.6-sol": { + "use_bedrock_runtime_chat_completions": true, + "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "cache_creation_input_token_cost": 5e-06, @@ -58499,6 +58505,8 @@ "supports_vision": true }, "us.openai.gpt-5.6-terra": { + "use_bedrock_runtime_chat_completions": true, + "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, "input_cost_per_token": 2.2e-06, "input_cost_per_token_above_272k_tokens": 4.4e-06, "cache_creation_input_token_cost": 2.75e-06, @@ -58528,6 +58536,8 @@ "supports_vision": true }, "global.openai.gpt-5.6-terra": { + "use_bedrock_runtime_chat_completions": true, + "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "cache_creation_input_token_cost": 2.5e-06, @@ -58557,6 +58567,8 @@ "supports_vision": true }, "us.openai.gpt-5.6-luna": { + "use_bedrock_runtime_chat_completions": true, + "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, "cache_creation_input_token_cost": 2.75e-07, @@ -58586,6 +58598,8 @@ "supports_vision": true }, "global.openai.gpt-5.6-luna": { + "use_bedrock_runtime_chat_completions": true, + "bedrock_runtime_chat_completions_tools_require_reasoning_none": true, "input_cost_per_token": 2e-07, "input_cost_per_token_above_272k_tokens": 4e-07, "cache_creation_input_token_cost": 2.5e-07, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index eca89e23887..9db1c8eedc4 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -74,6 +74,9 @@ "xhigh" ] }, + "bedrock_runtime_chat_completions_tools_require_reasoning_none": { + "type": "boolean" + }, "cache_creation_input_audio_token_cost": { "type": "number", "minimum": 0 diff --git a/tests/test_litellm/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/test_litellm/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 7724b3f0a52..230e323ace3 100644 --- a/tests/test_litellm/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -1,7 +1,6 @@ -"""Native Bedrock Runtime Chat Completions: Grok stays on /openai/v1/chat/completions.""" +"""Native Bedrock Runtime Chat Completions: Grok, gpt-oss and GPT-5.6 stay on /openai/v1/chat/completions.""" import json -from unittest.mock import patch import httpx import pytest @@ -9,25 +8,28 @@ import pytest import litellm from litellm.llms.bedrock.chat.chat_completions.transformation import ( AmazonBedrockRuntimeChatCompletionsConfig, + BedrockRuntimeChatCompletionsStreamingHandler, + ReasoningTagSplitter, + split_reasoning_tag, + with_max_completion_tokens, ) from litellm.llms.bedrock.common_utils import ( + BEDROCK_CONVERSE_ONLY_REQUEST_KEYS, BedrockModelInfo, + bedrock_request_needs_converse, get_bedrock_chat_config, uses_bedrock_runtime_chat_completions, ) +from litellm.llms.custom_httpx.http_handler import HTTPHandler @pytest.fixture def local_cost_map(monkeypatch): - original_model_cost = litellm.model_cost - try: - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true") - litellm.model_cost = litellm.get_model_cost_map(url="") - litellm.get_model_info.cache_clear() - yield - finally: - litellm.model_cost = original_model_cost - litellm.get_model_info.cache_clear() + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() @pytest.mark.parametrize( @@ -103,7 +105,27 @@ def test_transform_request_is_openai_chat_body_not_converse(): assert "messages" in body -def test_completion_posts_runtime_chat_completions(local_cost_map, monkeypatch): +def _chat_completion_json(content, model, tool_calls=None): + message = {"role": "assistant", "content": content, **({"tool_calls": tool_calls} if tool_calls else {})} + return { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1733529600, + "model": model, + "choices": [{"index": 0, "message": message, "finish_reason": "tool_calls" if tool_calls else "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + + +CONVERSE_JSON = { + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, +} + + +@pytest.fixture +def fake_aws_env(monkeypatch): monkeypatch.setenv("AWS_REGION_NAME", "us-west-2") monkeypatch.delenv("AWS_BEDROCK_RUNTIME_ENDPOINT", raising=False) monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) @@ -111,40 +133,490 @@ def test_completion_posts_runtime_chat_completions(local_cost_map, monkeypatch): monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") monkeypatch.setenv("AWS_SESSION_TOKEN", "testing") - requests: list[dict] = [] - def mock_post(self, url, data=None, json=None, headers=None, **kwargs): - requests.append({"url": url, "data": data, "json": json, "headers": headers or {}}) - return httpx.Response( - status_code=200, - json={ - "id": "chatcmpl-test", - "object": "chat.completion", - "created": 1733529600, - "model": "us.xai.grok-4.6", - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "ok"}, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, - }, - request=httpx.Request("POST", url), - ) +def _recording_client(**response_kwargs): + requests: list[httpx.Request] = [] - with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post): - response = litellm.completion( - model="us.xai.grok-4.6", - messages=[{"role": "user", "content": "hello"}], - ) + def handle(request): + requests.append(request) + return httpx.Response(200, **response_kwargs) + + return requests, HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle))) + + +def test_completion_posts_runtime_chat_completions(local_cost_map, fake_aws_env): + requests, client = _recording_client(json=_chat_completion_json("ok", "us.xai.grok-4.6")) + response = litellm.completion( + model="us.xai.grok-4.6", + messages=[{"role": "user", "content": "hello"}], + client=client, + ) assert response.choices[0].message.content == "ok" assert len(requests) == 1 - assert requests[0]["url"] == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions" - raw = requests[0]["data"] - body = json.loads(raw) if isinstance(raw, (str, bytes, bytearray)) else (requests[0]["json"] or {}) + assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions" + body = json.loads(requests[0].content) assert body["model"] == "us.xai.grok-4.6" assert body["messages"] == [{"role": "user", "content": "hello"}] assert "inferenceConfig" not in body + + +OPENAI_RUNTIME_MODELS = ( + "openai.gpt-oss-20b-1:0", + "openai.gpt-oss-120b-1:0", + "us.openai.gpt-5.6-sol", + "global.openai.gpt-5.6-sol", + "us.openai.gpt-5.6-terra", + "global.openai.gpt-5.6-terra", + "us.openai.gpt-5.6-luna", + "global.openai.gpt-5.6-luna", +) +GET_WEATHER_TOOL = { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, +} + + +@pytest.mark.parametrize("model", [*OPENAI_RUNTIME_MODELS, "bedrock/openai.gpt-oss-20b-1:0"]) +def test_openai_runtime_models_use_chat_completions_route(local_cost_map, model): + assert uses_bedrock_runtime_chat_completions(model) is True + assert BedrockModelInfo.get_bedrock_route(model) == "chat_completions" + assert isinstance(get_bedrock_chat_config(model), AmazonBedrockRuntimeChatCompletionsConfig) + + +@pytest.mark.parametrize("model", ["us.amazon.nova-micro-v1:0", "us.anthropic.claude-haiku-4-5-20251001-v1:0"]) +def test_nova_and_claude_stay_on_converse(local_cost_map, model): + assert uses_bedrock_runtime_chat_completions(model) is False + assert BedrockModelInfo.get_bedrock_route(model, {"tools": [GET_WEATHER_TOOL]}) == "converse" + + +@pytest.mark.parametrize("model", ["openai.gpt-oss-20b-1:0", "global.openai.gpt-5.6-sol"]) +def test_guardrail_config_falls_back_to_converse(local_cost_map, model): + guardrail = {"guardrailIdentifier": "gr-1", "guardrailVersion": "1"} + assert bedrock_request_needs_converse(model, {"guardrailConfig": guardrail}) is True + assert BedrockModelInfo.get_bedrock_route(model, {"guardrailConfig": guardrail}) == "converse" + assert BedrockModelInfo.get_bedrock_route(model, {"guardrailConfig": None}) == "chat_completions" + + +@pytest.mark.parametrize( + "request_params, expected_route", + [ + ({"tools": [GET_WEATHER_TOOL]}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": None}, "converse"), + ({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "none"}, "chat_completions"), + ({"reasoning_effort": "low"}, "chat_completions"), + ({"tools": None, "reasoning_effort": "low"}, "chat_completions"), + ({"tools": [], "reasoning_effort": "low"}, "chat_completions"), + ({}, "chat_completions"), + ], +) +def test_gpt56_tools_need_reasoning_none_on_chat_completions(local_cost_map, request_params, expected_route): + assert BedrockModelInfo.get_bedrock_route("global.openai.gpt-5.6-sol", request_params) == expected_route + assert BedrockModelInfo.get_bedrock_route("bedrock/us.openai.gpt-5.6-terra", request_params) == expected_route + + +@pytest.mark.parametrize("reasoning_effort", ["low", "high", None]) +def test_gpt_oss_tools_with_any_reasoning_effort_stay_on_chat_completions(local_cost_map, reasoning_effort): + params = {"tools": [GET_WEATHER_TOOL], "reasoning_effort": reasoning_effort} + assert bedrock_request_needs_converse("openai.gpt-oss-120b-1:0", params) is False + assert BedrockModelInfo.get_bedrock_route("openai.gpt-oss-120b-1:0", params) == "chat_completions" + + +def test_explicit_converse_prefix_wins_for_openai_models(local_cost_map): + assert BedrockModelInfo.get_bedrock_route("bedrock/converse/openai.gpt-oss-20b-1:0") == "converse" + assert BedrockModelInfo.get_bedrock_route("converse/global.openai.gpt-5.6-sol", {}) == "converse" + + +def test_map_openai_params_sends_max_tokens_as_max_completion_tokens(): + cfg = AmazonBedrockRuntimeChatCompletionsConfig() + mapped = cfg.map_openai_params( + non_default_params={"max_tokens": 64, "temperature": 0.1}, + optional_params={}, + model="global.openai.gpt-5.6-sol", + drop_params=False, + ) + assert mapped == {"max_completion_tokens": 64, "temperature": 0.1} + + +def test_map_openai_params_keeps_explicit_max_completion_tokens(): + cfg = AmazonBedrockRuntimeChatCompletionsConfig() + mapped = cfg.map_openai_params( + non_default_params={"max_tokens": 64, "max_completion_tokens": 32}, + optional_params={}, + model="openai.gpt-oss-20b-1:0", + drop_params=False, + ) + assert mapped == {"max_completion_tokens": 32} + + +def test_with_max_completion_tokens_leaves_other_params_alone(): + assert with_max_completion_tokens({"temperature": 0.5}) == {"temperature": 0.5} + + +def test_supported_params_include_reasoning_effort_for_gpt56(local_cost_map): + cfg = AmazonBedrockRuntimeChatCompletionsConfig() + assert "reasoning_effort" in cfg.get_supported_openai_params("global.openai.gpt-5.6-sol") + assert "reasoning_effort" in cfg.get_supported_openai_params("openai.gpt-oss-20b-1:0") + + +def test_split_reasoning_tag_splits_leading_tag(): + assert split_reasoning_tag("plan it\n\n\nHello") == ("plan it\n", "Hello") + + +def test_split_reasoning_tag_drops_an_empty_tag(): + assert split_reasoning_tag("Hello") == (None, "Hello") + + +@pytest.mark.parametrize( + "content", + [ + "plan it\n\n\nHello", + "never closed", + "later", + "", + ], +) +@pytest.mark.parametrize("chunk_size", [1, 3, 7]) +def test_split_reasoning_tag_matches_the_streamed_split(content, chunk_size): + chunks = [content[start : start + chunk_size] for start in range(0, len(content), chunk_size)] + streamed_reasoning, streamed_content = _run_splitter(chunks) + + assert split_reasoning_tag(content) == (streamed_reasoning or None, streamed_content) + + +def test_split_reasoning_tag_passes_plain_content_through(): + assert split_reasoning_tag("Hello") == (None, "Hello") + + +def test_split_reasoning_tag_ignores_tag_after_content_starts(): + content = "Hello not mine" + assert split_reasoning_tag(content) == (None, content) + + +def _run_splitter(chunks): + state = ReasoningTagSplitter() + reasoning = "" + content = "" + for chunk in chunks: + state, fed_reasoning, fed_content = state.feed(chunk) + reasoning += fed_reasoning + content += fed_content + state, flushed_reasoning, flushed_content = state.flush() + return reasoning + flushed_reasoning, content + flushed_content + + +def test_reasoning_tag_splitter_handles_tags_split_across_chunks(): + assert _run_splitter(["I think", " so\n\nHel", "lo"]) == ("I think so", "Hello") + + +def test_reasoning_tag_splitter_passes_plain_content_through(): + assert _run_splitter(["Hel", "lo later"]) == ("", "Hello later") + + +def test_reasoning_tag_splitter_flushes_unclosed_reasoning(): + assert _run_splitter(["never clo", "sed"]) == ("never closed", "") + + +def test_reasoning_tag_splitter_releases_a_false_tag_prefix(): + assert _run_splitter(["<", "b>x"]) == ("", "x") + + +def _stream_chunk(delta, finish_reason=None, index=0): + return { + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "created": 1733529600, + "model": "openai.gpt-oss-20b-1:0", + "choices": [{"index": index, "delta": delta, "finish_reason": finish_reason}], + } + + +def test_streaming_handler_splits_reasoning_deltas_per_choice(): + handler = BedrockRuntimeChatCompletionsStreamingHandler(streaming_response=iter(()), sync_stream=True) + + first = handler.chunk_parser(_stream_chunk({"role": "assistant", "content": "I think"})) + assert first.choices[0].delta.reasoning_content == "I think" + assert not first.choices[0].delta.content + + second = handler.chunk_parser(_stream_chunk({"content": " so\n\nHello"})) + assert second.choices[0].delta.reasoning_content == " so" + assert second.choices[0].delta.content == "Hello" + + tool_call = {"index": 0, "id": "call_0", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}} + third = handler.chunk_parser(_stream_chunk({"content": None, "tool_calls": [tool_call]})) + assert third.choices[0].delta.tool_calls[0].function.name == "get_weather" + + last = handler.chunk_parser(_stream_chunk({}, finish_reason="stop")) + assert last.choices[0].finish_reason == "stop" + + +def _reasoning_of(parsed): + return getattr(parsed.choices[0].delta, "reasoning_content", None) + + +def test_streaming_handler_keeps_split_state_per_choice_index(): + handler = BedrockRuntimeChatCompletionsStreamingHandler(streaming_response=iter(()), sync_stream=True) + + opened = handler.chunk_parser(_stream_chunk({"content": "first"}, index=0)) + assert _reasoning_of(opened) == "first" + + plain = handler.chunk_parser(_stream_chunk({"content": "plain answer"}, index=1)) + assert _reasoning_of(plain) is None + assert plain.choices[0].delta.content == "plain answer" + + still_reasoning = handler.chunk_parser(_stream_chunk({"content": " more"}, index=0)) + assert _reasoning_of(still_reasoning) == " more" + assert not still_reasoning.choices[0].delta.content + + +def test_streaming_handler_flushes_held_text_on_an_empty_final_delta(): + handler = BedrockRuntimeChatCompletionsStreamingHandler(streaming_response=iter(()), sync_stream=True) + + held = handler.chunk_parser(_stream_chunk({"content": "almost doneplan\n\nHi", "openai.gpt-oss-20b-1:0") + ) + response = litellm.completion( + model="bedrock/openai.gpt-oss-20b-1:0", + messages=[{"role": "user", "content": "hello"}], + max_tokens=64, + reasoning_effort="low", + tools=[GET_WEATHER_TOOL], + client=client, + ) + + assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions" + body = json.loads(requests[0].content) + assert body["model"] == "openai.gpt-oss-20b-1:0" + assert body["max_completion_tokens"] == 64 + assert "max_tokens" not in body + assert body["reasoning_effort"] == "low" + assert body["tools"] == [GET_WEATHER_TOOL] + assert response.choices[0].message.reasoning_content == "plan" + assert response.choices[0].message.content == "Hi" + + +def test_gpt56_tools_with_reasoning_effort_go_to_converse(local_cost_map, fake_aws_env): + requests, client = _recording_client(json=CONVERSE_JSON) + response = litellm.completion( + model="bedrock/global.openai.gpt-5.6-sol", + messages=[{"role": "user", "content": "hello"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + client=client, + ) + + assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-5.6-sol/converse") + assert json.loads(requests[0].content)["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather" + assert response.choices[0].message.content == "ok" + + +def test_gpt56_tools_with_reasoning_none_stay_on_chat_completions(local_cost_map, fake_aws_env): + tool_calls = [ + {"id": "call_0", "type": "function", "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}} + ] + requests, client = _recording_client(json=_chat_completion_json(None, "global.openai.gpt-5.6-sol", tool_calls)) + response = litellm.completion( + model="bedrock/global.openai.gpt-5.6-sol", + messages=[{"role": "user", "content": "weather in Paris"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="none", + max_tokens=64, + client=client, + ) + + assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions" + body = json.loads(requests[0].content) + assert body["tools"] == [GET_WEATHER_TOOL] + assert body["reasoning_effort"] == "none" + assert body["max_completion_tokens"] == 64 + assert response.choices[0].message.tool_calls[0].function.name == "get_weather" + + +@pytest.mark.parametrize( + "converse_only_param", + [ + {"guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "1"}}, + {"performanceConfig": {"latency": "optimized"}}, + {"requestMetadata": {"team": "search"}}, + {"serviceTier": {"type": "priority"}}, + ], + ids=lambda param: next(iter(param)), +) +def test_converse_only_request_keys_go_to_converse(local_cost_map, fake_aws_env, converse_only_param): + requests, client = _recording_client(json=CONVERSE_JSON) + litellm.completion( + model="bedrock/openai.gpt-oss-20b-1:0", + messages=[{"role": "user", "content": "hello"}], + client=client, + **converse_only_param, + ) + + assert requests[0].url.raw_path.endswith(b"/model/openai.gpt-oss-20b-1%3A0/converse") + (key, value), = converse_only_param.items() + assert json.loads(requests[0].content)[key] == value + + +def test_converse_only_keys_cover_every_converse_config_block(): + assert set(litellm.AmazonConverseConfig.get_config_blocks()) <= BEDROCK_CONVERSE_ONLY_REQUEST_KEYS + + +def test_operator_owned_request_metadata_goes_to_converse(local_cost_map, fake_aws_env, monkeypatch): + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ["user_api_key_team_alias"]) + requests, client = _recording_client(json=CONVERSE_JSON) + litellm.completion( + model="bedrock/openai.gpt-oss-20b-1:0", + messages=[{"role": "user", "content": "hello"}], + metadata={"user_api_key_team_alias": "search"}, + client=client, + ) + + assert requests[0].url.raw_path.endswith(b"/model/openai.gpt-oss-20b-1%3A0/converse") + assert json.loads(requests[0].content)["requestMetadata"] == {"user_api_key_team_alias": "search"} + + +def test_dropped_converse_only_key_keeps_the_request_on_chat_completions(local_cost_map, fake_aws_env): + requests, client = _recording_client(json=_chat_completion_json("ok", "openai.gpt-oss-20b-1:0")) + litellm.completion( + model="bedrock/openai.gpt-oss-20b-1:0", + messages=[{"role": "user", "content": "hello"}], + guardrailConfig={"guardrailIdentifier": "gr-1", "guardrailVersion": "1"}, + additional_drop_params=["guardrailConfig"], + max_tokens=8, + client=client, + ) + + assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions" + body = json.loads(requests[0].content) + assert "guardrailConfig" not in body + assert body["max_completion_tokens"] == 8 + assert "inferenceConfig" not in body + + +def test_dropped_tools_keep_gpt56_reasoning_request_on_chat_completions(local_cost_map, fake_aws_env): + requests, client = _recording_client(json=_chat_completion_json("ok", "global.openai.gpt-5.6-sol")) + litellm.completion( + model="bedrock/global.openai.gpt-5.6-sol", + messages=[{"role": "user", "content": "hello"}], + tools=[GET_WEATHER_TOOL], + reasoning_effort="low", + additional_drop_params=["tools"], + client=client, + ) + + assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions" + body = json.loads(requests[0].content) + assert "tools" not in body + assert body["reasoning_effort"] == "low" + + +def test_legacy_functions_stay_on_chat_completions(local_cost_map, fake_aws_env): + requests, client = _recording_client(json=_chat_completion_json("ok", "openai.gpt-oss-20b-1:0")) + litellm.completion( + model="bedrock/openai.gpt-oss-20b-1:0", + messages=[{"role": "user", "content": "hello"}], + functions=[GET_WEATHER_TOOL["function"]], + client=client, + ) + + assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions" + assert json.loads(requests[0].content)["functions"] == [GET_WEATHER_TOOL["function"]] + + +def test_converse_fallback_validates_against_converse_params(local_cost_map, fake_aws_env): + requests, client = _recording_client(json=CONVERSE_JSON) + guardrail = {"guardrailIdentifier": "gr-1", "guardrailVersion": "1"} + with pytest.raises(litellm.UnsupportedParamsError, match="seed"): + litellm.completion( + model="bedrock/openai.gpt-oss-20b-1:0", + messages=[{"role": "user", "content": "hello"}], + guardrailConfig=guardrail, + seed=7, + client=client, + ) + litellm.completion( + model="bedrock/openai.gpt-oss-20b-1:0", + messages=[{"role": "user", "content": "hello"}], + guardrailConfig=guardrail, + seed=7, + drop_params=True, + client=client, + ) + + assert requests[0].url.raw_path.endswith(b"/model/openai.gpt-oss-20b-1%3A0/converse") + assert "seed" not in json.loads(requests[0].content) + + +def test_n_is_rejected_before_reaching_chat_completions(local_cost_map, fake_aws_env): + requests, client = _recording_client(json=_chat_completion_json("ok", "openai.gpt-oss-20b-1:0")) + with pytest.raises(litellm.UnsupportedParamsError, match="'n'"): + litellm.completion( + model="bedrock/openai.gpt-oss-20b-1:0", + messages=[{"role": "user", "content": "hello"}], + n=2, + client=client, + ) + litellm.completion( + model="bedrock/openai.gpt-oss-20b-1:0", + messages=[{"role": "user", "content": "hello"}], + n=2, + drop_params=True, + client=client, + ) + + assert "n" not in json.loads(requests[0].content) + + +def _sse(chunks): + return ("".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n").encode() + + +def test_gpt_oss_streaming_completion_splits_reasoning(local_cost_map, fake_aws_env): + chunks = ( + _stream_chunk({"role": "assistant", "content": "plan"}), + _stream_chunk({"content": "\n\nHi"}), + _stream_chunk({}, finish_reason="stop"), + ) + requests, client = _recording_client(content=_sse(chunks), headers={"content-type": "text/event-stream"}) + stream = litellm.completion( + model="bedrock/openai.gpt-oss-20b-1:0", + messages=[{"role": "user", "content": "hello"}], + stream=True, + client=client, + ) + deltas = [chunk.choices[0].delta for chunk in stream] + + assert [str(request.url) for request in requests] == [ + "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions" + ] + assert json.loads(requests[0].content)["stream"] is True + assert "".join(getattr(delta, "reasoning_content", None) or "" for delta in deltas) == "plan" + assert "".join(delta.content or "" for delta in deltas) == "Hi" + + +def test_streaming_handler_keeps_native_reasoning_next_to_the_tagged_split(): + handler = BedrockRuntimeChatCompletionsStreamingHandler(streaming_response=iter(()), sync_stream=True) + parsed = handler.chunk_parser( + _stream_chunk({"reasoning": "native ", "content": "taggedHi"}, finish_reason="stop") + ) + + assert parsed.choices[0].delta.reasoning_content == "native tagged" + assert parsed.choices[0].delta.content == "Hi" diff --git a/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py b/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py index aa0827c5ae5..a6b8ba1da1d 100644 --- a/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py +++ b/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py @@ -138,9 +138,12 @@ def _bedrock_response(model, usage): @pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id) -def test_bedrock_gpt_5_6_profiles_route_to_converse(profile, local_model_cost_map): - """GPT-5.6 is served by Converse on bedrock-runtime, never by Invoke.""" - assert BedrockModelInfo.get_bedrock_route(f"bedrock/{profile.model_id}") == "converse" +def test_bedrock_gpt_5_6_profiles_never_route_to_invoke(profile, local_model_cost_map): + """GPT-5.6 is served by bedrock-runtime's native Chat Completions, and by Converse when + the request carries function tools without reasoning_effort "none", never by Invoke.""" + assert BedrockModelInfo.get_bedrock_route(f"bedrock/{profile.model_id}") == "chat_completions" + tools_with_reasoning = {"tools": [{"type": "function", "function": {"name": "f"}}], "reasoning_effort": "low"} + assert BedrockModelInfo.get_bedrock_route(f"bedrock/{profile.model_id}", tools_with_reasoning) == "converse" @pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 3336ad6d33a..963bef1114a 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -878,6 +878,8 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_video_input": {"type": "boolean"}, "supports_vision": {"type": "boolean"}, "supports_web_search": {"type": "boolean"}, + "use_bedrock_runtime_chat_completions": {"type": "boolean"}, + "bedrock_runtime_chat_completions_tools_require_reasoning_none": {"type": "boolean"}, "supports_url_context": {"type": "boolean"}, "supports_multimodal": {"type": "boolean"}, "uses_embed_content": {"type": "boolean"},