mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-26 01:12:21 +00:00
feat(bedrock): serve gpt-oss and gpt-5.6 chat completions on runtime's native openai path
This commit is contained in:
mateo-berri
parent
df7e9daf00
commit
6b9f067c80
11 changed files with 897 additions and 128 deletions
|
|
@ -31,6 +31,7 @@ EXTRA_BOOLEAN_KEYS = frozenset(
|
|||
"uses_embed_content",
|
||||
"use_openai_responses_path",
|
||||
"use_bedrock_runtime_chat_completions",
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none",
|
||||
"bedrock_converse_supports_strict_tools",
|
||||
"thinking_always_on",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,30 +2,170 @@
|
|||
Native OpenAI Chat Completions on Amazon Bedrock Runtime.
|
||||
|
||||
AWS serves this surface at
|
||||
``https://bedrock-runtime.{region}.amazonaws.com/openai/v1/chat/completions``.
|
||||
Grok 4.6 on runtime is one of the models that uses it: chat completions stay
|
||||
chat completions instead of being rewritten to Converse.
|
||||
``https://bedrock-runtime.{region}.amazonaws.com/openai/v1/chat/completions``
|
||||
for the models whose price-map entry sets ``use_bedrock_runtime_chat_completions``
|
||||
(Grok 4.6, gpt-oss, the GPT-5.6 family): chat completions stay chat completions
|
||||
instead of being rewritten to Converse.
|
||||
|
||||
Usage: model="us.xai.grok-4.6" or model="bedrock/us.xai.grok-4.6"
|
||||
Explicit ``bedrock/converse/...`` still uses Converse.
|
||||
Usage: model="us.xai.grok-4.6", model="bedrock/openai.gpt-oss-20b-1:0" or
|
||||
model="bedrock/global.openai.gpt-5.6-sol". Explicit ``bedrock/converse/...``
|
||||
still uses Converse, and so does a request that needs a Converse-only feature
|
||||
(``bedrock_request_needs_converse`` in ``common_utils``).
|
||||
"""
|
||||
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from typing import Any, Final
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.common_utils import BedrockError, strip_bedrock_routing_prefix
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler
|
||||
from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import Choices, ModelResponse, ModelResponseStream
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
REASONING_OPEN_TAG: Final = "<reasoning>"
|
||||
REASONING_CLOSE_TAG: Final = "</reasoning>"
|
||||
|
||||
|
||||
def _held_close_tag_prefix(text: str) -> int:
|
||||
return next(
|
||||
(
|
||||
size
|
||||
for size in range(min(len(text), len(REASONING_CLOSE_TAG) - 1), 0, -1)
|
||||
if REASONING_CLOSE_TAG.startswith(text[-size:])
|
||||
),
|
||||
0,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReasoningTagSplitter:
|
||||
"""
|
||||
The same split for a stream of content deltas, where a tag can arrive across chunks.
|
||||
|
||||
``feed`` returns the next state plus the reasoning and content text the delta contributes;
|
||||
``flush`` releases what the stream ended on before a tag resolved.
|
||||
"""
|
||||
|
||||
phase: Literal["start", "reasoning", "after_close", "content"] = "start"
|
||||
pending: str = ""
|
||||
|
||||
def feed(self, text: str) -> tuple["ReasoningTagSplitter", str, str]:
|
||||
match self.phase:
|
||||
case "content":
|
||||
return self, "", text
|
||||
case "after_close":
|
||||
content: Final = text.lstrip()
|
||||
return (replace(self, phase="content") if content else self), "", content
|
||||
case "start":
|
||||
return self._feed_start(self.pending + text)
|
||||
case "reasoning":
|
||||
return self._feed_reasoning(self.pending + text)
|
||||
|
||||
def _feed_start(self, buffered: str) -> tuple["ReasoningTagSplitter", str, str]:
|
||||
if buffered.startswith(REASONING_OPEN_TAG):
|
||||
return replace(self, phase="reasoning", pending="")._feed_reasoning(buffered[len(REASONING_OPEN_TAG) :])
|
||||
if REASONING_OPEN_TAG.startswith(buffered):
|
||||
return replace(self, pending=buffered), "", ""
|
||||
return replace(self, phase="content", pending=""), "", buffered
|
||||
|
||||
def _feed_reasoning(self, buffered: str) -> tuple["ReasoningTagSplitter", str, str]:
|
||||
close_at: Final = buffered.find(REASONING_CLOSE_TAG)
|
||||
if close_at >= 0:
|
||||
after_close: Final = replace(self, phase="after_close", pending="")
|
||||
next_state, _, content = after_close.feed(buffered[close_at + len(REASONING_CLOSE_TAG) :])
|
||||
return next_state, buffered[:close_at], content
|
||||
held: Final = _held_close_tag_prefix(buffered)
|
||||
return replace(self, pending=buffered[len(buffered) - held :]), buffered[: len(buffered) - held], ""
|
||||
|
||||
def flush(self) -> tuple["ReasoningTagSplitter", str, str]:
|
||||
drained: Final = replace(self, phase="content", pending="")
|
||||
if self.phase == "reasoning":
|
||||
return drained, self.pending, ""
|
||||
return drained, "", self.pending
|
||||
|
||||
|
||||
def _split_streamed_content(
|
||||
splitter: ReasoningTagSplitter, content: str | None, finished: bool
|
||||
) -> tuple[ReasoningTagSplitter, str, str]:
|
||||
fed_state, fed_reasoning, fed_content = splitter.feed(content or "")
|
||||
if not finished:
|
||||
return fed_state, fed_reasoning, fed_content
|
||||
drained, flushed_reasoning, flushed_content = fed_state.flush()
|
||||
return drained, fed_reasoning + flushed_reasoning, fed_content + flushed_content
|
||||
|
||||
|
||||
def split_reasoning_tag(content: str) -> tuple[str | None, str]:
|
||||
"""
|
||||
Split gpt-oss's inline ``<reasoning>...</reasoning>`` prefix out of a complete message.
|
||||
|
||||
Runs the streaming splitter over the whole message, so a streamed and a non-streamed
|
||||
response to the same completion split identically. Returns ``(None, content)`` when the
|
||||
message does not start with the tag.
|
||||
"""
|
||||
_, reasoning, body = _split_streamed_content(ReasoningTagSplitter(), content, finished=True)
|
||||
return reasoning or None, body
|
||||
|
||||
|
||||
class BedrockRuntimeChatCompletionsStreamingHandler(OpenAIChatCompletionStreamingHandler):
|
||||
"""OpenAI chunk parsing plus the ``<reasoning>`` split, tracked per choice index."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse,
|
||||
sync_stream: bool,
|
||||
json_mode: bool | None = False,
|
||||
) -> None:
|
||||
super().__init__(streaming_response=streaming_response, sync_stream=sync_stream, json_mode=json_mode)
|
||||
self._splitters: Mapping[int, ReasoningTagSplitter] = MappingProxyType({})
|
||||
|
||||
def chunk_parser(self, chunk: dict) -> ModelResponseStream: # mutable-ok: BaseModelResponseIterator signature
|
||||
parsed: Final = super().chunk_parser(chunk)
|
||||
for choice in parsed.choices:
|
||||
next_state, reasoning, content = _split_streamed_content(
|
||||
self._splitters.get(choice.index, ReasoningTagSplitter()),
|
||||
choice.delta.content,
|
||||
choice.finish_reason is not None,
|
||||
)
|
||||
self._splitters = MappingProxyType({**self._splitters, choice.index: next_state})
|
||||
if reasoning:
|
||||
choice.delta.reasoning_content = f"{getattr(choice.delta, 'reasoning_content', None) or ''}{reasoning}"
|
||||
if content or choice.delta.content is not None:
|
||||
choice.delta.content = content
|
||||
return parsed
|
||||
|
||||
|
||||
def with_max_completion_tokens(params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""
|
||||
Send the caller's ``max_tokens`` as ``max_completion_tokens``.
|
||||
|
||||
Every model on this surface accepts ``max_completion_tokens`` and the GPT-5.6 family
|
||||
rejects ``max_tokens``; an explicit ``max_completion_tokens`` wins when both are set.
|
||||
"""
|
||||
if "max_tokens" not in params:
|
||||
return params
|
||||
return MappingProxyType(
|
||||
{
|
||||
key: value
|
||||
for key, value in (("max_completion_tokens", params["max_tokens"]), *params.items())
|
||||
if key != "max_tokens"
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
|
||||
def __init__(self, aws_signer: BaseAWSLLM | None = None):
|
||||
def __init__(self, aws_signer: BaseAWSLLM | None = None) -> None:
|
||||
super().__init__()
|
||||
self._aws_signer: Final = aws_signer or BaseAWSLLM()
|
||||
|
||||
|
|
@ -34,7 +174,10 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
|
|||
return "bedrock"
|
||||
|
||||
def get_error_class(
|
||||
self, error_message: str, status_code: int, headers: dict[str, object] | httpx.Headers
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict[str, object] | httpx.Headers, # mutable-ok: BaseConfig signature
|
||||
) -> BaseLLMException:
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
||||
|
|
@ -43,13 +186,15 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
|
|||
api_base: str | None,
|
||||
api_key: str | None,
|
||||
model: str,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
optional_params: dict, # mutable-ok: BaseConfig signature
|
||||
litellm_params: dict, # mutable-ok: BaseConfig signature
|
||||
stream: bool | None = None,
|
||||
) -> str:
|
||||
if api_base is not None and "chat/completions" in api_base:
|
||||
return api_base.rstrip("/")
|
||||
aws_region_name: Final = self._aws_signer._get_aws_region_name(optional_params=optional_params, model=model)
|
||||
aws_region_name: Final = self._aws_signer._get_aws_region_name( # pyright: ignore[reportPrivateUsage] # BaseAWSLLM has no public region resolver
|
||||
optional_params=optional_params, model=model
|
||||
)
|
||||
endpoint_url, _ = self._aws_signer.get_runtime_endpoint(
|
||||
api_base=api_base,
|
||||
aws_bedrock_runtime_endpoint=optional_params.get("aws_bedrock_runtime_endpoint"),
|
||||
|
|
@ -64,16 +209,16 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
|
|||
|
||||
def sign_request(
|
||||
self,
|
||||
headers: dict,
|
||||
optional_params: dict,
|
||||
request_data: dict,
|
||||
headers: dict, # mutable-ok: BaseConfig signature
|
||||
optional_params: dict, # mutable-ok: BaseConfig signature
|
||||
request_data: dict, # mutable-ok: BaseConfig signature
|
||||
api_base: str,
|
||||
api_key: str | None = None,
|
||||
model: str | None = None,
|
||||
stream: bool | None = None,
|
||||
fake_stream: bool | None = None,
|
||||
) -> tuple[dict, bytes | None]:
|
||||
return self._aws_signer._sign_request(
|
||||
) -> tuple[dict, bytes | None]: # mutable-ok: BaseConfig signature
|
||||
return self._aws_signer._sign_request( # pyright: ignore[reportPrivateUsage] # BaseAWSLLM has no public signer
|
||||
service_name="bedrock",
|
||||
headers=headers,
|
||||
optional_params=optional_params,
|
||||
|
|
@ -85,21 +230,44 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
|
|||
fake_stream=fake_stream,
|
||||
)
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict, # mutable-ok: BaseConfig signature
|
||||
optional_params: dict, # mutable-ok: BaseConfig signature
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
replace_max_completion_tokens_with_max_tokens: bool = False,
|
||||
) -> dict: # mutable-ok: BaseConfig signature
|
||||
mapped: Final = super().map_openai_params(
|
||||
non_default_params=non_default_params,
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
drop_params=drop_params,
|
||||
replace_max_completion_tokens_with_max_tokens=replace_max_completion_tokens_with_max_tokens,
|
||||
)
|
||||
return dict(with_max_completion_tokens(mapped)) # mutable-ok: get_optional_params keeps filling this dict
|
||||
|
||||
def _inference_params(
|
||||
self, optional_params: Mapping[str, object]
|
||||
) -> dict[str, object]: # mutable-ok: BaseConfig signature of transform_request
|
||||
return { # mutable-ok: OpenAILikeChatConfig.transform_request takes a plain dict
|
||||
key: value
|
||||
for key, value in optional_params.items()
|
||||
if key not in self._aws_signer.aws_authentication_params
|
||||
}
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
inference_params: Final = {
|
||||
k: v for k, v in optional_params.items() if k not in self._aws_signer.aws_authentication_params
|
||||
}
|
||||
messages: list[AllMessageValues], # mutable-ok: BaseConfig signature
|
||||
optional_params: dict, # mutable-ok: BaseConfig signature
|
||||
litellm_params: dict, # mutable-ok: BaseConfig signature
|
||||
headers: dict, # mutable-ok: BaseConfig signature
|
||||
) -> dict: # mutable-ok: BaseConfig signature
|
||||
return super().transform_request(
|
||||
model=strip_bedrock_routing_prefix(model),
|
||||
messages=messages,
|
||||
optional_params=inference_params,
|
||||
optional_params=self._inference_params(optional_params),
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
|
|
@ -107,33 +275,68 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
|
|||
async def async_transform_request(
|
||||
self,
|
||||
model: str,
|
||||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
inference_params: Final = {
|
||||
k: v for k, v in optional_params.items() if k not in self._aws_signer.aws_authentication_params
|
||||
}
|
||||
messages: list[AllMessageValues], # mutable-ok: BaseConfig signature
|
||||
optional_params: dict, # mutable-ok: BaseConfig signature
|
||||
litellm_params: dict, # mutable-ok: BaseConfig signature
|
||||
headers: dict, # mutable-ok: BaseConfig signature
|
||||
) -> dict: # mutable-ok: BaseConfig signature
|
||||
return await super().async_transform_request(
|
||||
model=strip_bedrock_routing_prefix(model),
|
||||
messages=messages,
|
||||
optional_params=inference_params,
|
||||
optional_params=self._inference_params(optional_params),
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
def transform_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: ModelResponse,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
request_data: dict, # mutable-ok: BaseConfig signature
|
||||
messages: list[AllMessageValues], # mutable-ok: BaseConfig signature
|
||||
optional_params: dict, # mutable-ok: BaseConfig signature
|
||||
litellm_params: dict, # mutable-ok: BaseConfig signature
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
response: Final = super().transform_response(
|
||||
model=model,
|
||||
raw_response=raw_response,
|
||||
model_response=model_response,
|
||||
logging_obj=logging_obj,
|
||||
request_data=request_data,
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
encoding=encoding,
|
||||
api_key=api_key,
|
||||
json_mode=json_mode,
|
||||
)
|
||||
for choice in response.choices:
|
||||
if not isinstance(choice, Choices) or not isinstance(choice.message.content, str):
|
||||
continue
|
||||
reasoning, content = split_reasoning_tag(choice.message.content)
|
||||
if reasoning is not None:
|
||||
choice.message.reasoning_content = (
|
||||
f"{getattr(choice.message, 'reasoning_content', None) or ''}{reasoning}"
|
||||
)
|
||||
choice.message.content = content
|
||||
return response
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
headers: dict, # mutable-ok: BaseConfig signature
|
||||
model: str,
|
||||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
messages: list[AllMessageValues], # mutable-ok: BaseConfig signature
|
||||
optional_params: dict, # mutable-ok: BaseConfig signature
|
||||
litellm_params: dict, # mutable-ok: BaseConfig signature
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict:
|
||||
headers = super().validate_environment(
|
||||
) -> dict: # mutable-ok: BaseConfig signature
|
||||
validated: Final = super().validate_environment(
|
||||
headers=headers,
|
||||
model=model,
|
||||
messages=messages,
|
||||
|
|
@ -143,31 +346,25 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
|
|||
api_base=api_base,
|
||||
)
|
||||
project_id: Final = litellm_params.get("aws_bedrock_project_id")
|
||||
if project_id:
|
||||
headers["OpenAI-Project"] = project_id
|
||||
return headers
|
||||
if not project_id:
|
||||
return validated
|
||||
return {**validated, "OpenAI-Project": project_id} # mutable-ok: BaseConfig signature returns a dict
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
base_params: Final = super().get_supported_openai_params(model)
|
||||
try:
|
||||
if litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider):
|
||||
if "reasoning_effort" not in base_params:
|
||||
base_params.append("reasoning_effort")
|
||||
except Exception as e:
|
||||
verbose_logger.debug("AmazonBedrockRuntimeChatCompletionsConfig: error checking reasoning support: %s", e)
|
||||
return base_params
|
||||
def get_supported_openai_params(self, model: str) -> list: # mutable-ok: BaseConfig signature
|
||||
base_params: Final = [param for param in super().get_supported_openai_params(model) if param != "n"]
|
||||
if "reasoning_effort" in base_params or not litellm.supports_reasoning(
|
||||
model=model, custom_llm_provider=self.custom_llm_provider
|
||||
):
|
||||
return base_params
|
||||
return [*base_params, "reasoning_effort"] # mutable-ok: BaseConfig signature returns a list
|
||||
|
||||
def get_model_response_iterator(
|
||||
self,
|
||||
streaming_response: Iterator[str] | AsyncIterator[str] | Any,
|
||||
streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse,
|
||||
sync_stream: bool,
|
||||
json_mode: bool | None = False,
|
||||
) -> Any:
|
||||
from litellm.llms.openai.chat.gpt_transformation import (
|
||||
OpenAIChatCompletionStreamingHandler,
|
||||
)
|
||||
|
||||
return OpenAIChatCompletionStreamingHandler(
|
||||
) -> BedrockRuntimeChatCompletionsStreamingHandler:
|
||||
return BedrockRuntimeChatCompletionsStreamingHandler(
|
||||
streaming_response=streaming_response,
|
||||
sync_stream=sync_stream,
|
||||
json_mode=json_mode,
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from litellm.llms.base_llm.anthropic_messages.transformation import (
|
|||
)
|
||||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.bedrock.request_metadata import bedrock_request_metadata_is_owned
|
||||
from litellm.secret_managers.main import get_secret, get_secret_str
|
||||
from litellm.types.llms.bedrock import AWS_AUTH_PARAM_KEYS, AwsAuthParams
|
||||
|
||||
|
|
@ -37,6 +38,18 @@ if TYPE_CHECKING:
|
|||
|
||||
_ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs"
|
||||
_OPENAI_FAMILY_MODEL_RE: Final = re.compile(r"(^|[./])openai\.")
|
||||
BedrockRoute = Literal[
|
||||
"converse",
|
||||
"invoke",
|
||||
"claude_platform",
|
||||
"converse_like",
|
||||
"agent",
|
||||
"agentcore",
|
||||
"async_invoke",
|
||||
"openai",
|
||||
"mantle",
|
||||
"chat_completions",
|
||||
]
|
||||
|
||||
|
||||
def error_response_text(response: httpx.Response) -> str:
|
||||
|
|
@ -782,17 +795,54 @@ def strip_bedrock_routing_prefix(model: str) -> str:
|
|||
return model
|
||||
|
||||
|
||||
def _bedrock_price_map_flag(model: str, flag: str) -> bool:
|
||||
entries: Final = (litellm.model_cost.get(key) for key in (model, strip_bedrock_routing_prefix(model)))
|
||||
return any(entry is not None and entry.get(flag) is True for entry in entries)
|
||||
|
||||
|
||||
def uses_bedrock_runtime_chat_completions(model: str) -> bool:
|
||||
"""Whether this Bedrock model should use runtime native Chat Completions.
|
||||
|
||||
Data-driven from the price-map ``use_bedrock_runtime_chat_completions`` flag
|
||||
so onboarding a model is a JSON change. Explicit ``converse/`` still wins in
|
||||
``get_bedrock_route`` because prefix routes are checked first.
|
||||
``get_bedrock_route`` because prefix routes are checked first, and a request
|
||||
that needs a Converse-only feature (``bedrock_request_needs_converse``) is
|
||||
served by Converse even on a flagged model.
|
||||
"""
|
||||
stripped: Final = strip_bedrock_routing_prefix(model)
|
||||
return any(
|
||||
(litellm.model_cost.get(key) or {}).get("use_bedrock_runtime_chat_completions") is True
|
||||
for key in (model, stripped)
|
||||
return _bedrock_price_map_flag(model, "use_bedrock_runtime_chat_completions")
|
||||
|
||||
|
||||
def bedrock_runtime_chat_completions_tools_require_reasoning_none(model: str) -> bool:
|
||||
"""Whether AWS's native Chat Completions only serves this model's function tools with ``reasoning_effort="none"``.
|
||||
|
||||
Data-driven from the price-map ``bedrock_runtime_chat_completions_tools_require_reasoning_none``
|
||||
flag (the GPT-5.6 family). Converse serves tools with any effort, so those requests fall back to it.
|
||||
"""
|
||||
return _bedrock_price_map_flag(model, "bedrock_runtime_chat_completions_tools_require_reasoning_none")
|
||||
|
||||
|
||||
BEDROCK_CONVERSE_ONLY_REQUEST_KEYS: Final = frozenset(
|
||||
("guardrailConfig", "performanceConfig", "serviceTier", "requestMetadata", "outputConfig")
|
||||
)
|
||||
|
||||
|
||||
def bedrock_request_needs_converse(model: str, request_params: Mapping[str, object]) -> bool:
|
||||
"""Whether a request on a runtime-Chat-Completions model must still be served by Converse.
|
||||
|
||||
Converse-shaped body keys (``BEDROCK_CONVERSE_ONLY_REQUEST_KEYS``) are rejected as malformed input by
|
||||
AWS's native OpenAI surface, operator-owned request metadata is only written onto the Converse body,
|
||||
and function tools on a ``bedrock_runtime_chat_completions_tools_require_reasoning_none`` model are
|
||||
rejected there unless ``reasoning_effort`` is exactly ``"none"``.
|
||||
"""
|
||||
if any(request_params.get(key) is not None for key in BEDROCK_CONVERSE_ONLY_REQUEST_KEYS):
|
||||
return True
|
||||
if bedrock_request_metadata_is_owned():
|
||||
return True
|
||||
if not request_params.get("tools"):
|
||||
return False
|
||||
return (
|
||||
bedrock_runtime_chat_completions_tools_require_reasoning_none(model)
|
||||
and request_params.get("reasoning_effort") != "none"
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1130,20 +1180,13 @@ class BedrockModelInfo(BaseLLMModelInfo):
|
|||
@staticmethod
|
||||
def get_bedrock_route(
|
||||
model: str,
|
||||
) -> Literal[
|
||||
"converse",
|
||||
"invoke",
|
||||
"claude_platform",
|
||||
"converse_like",
|
||||
"agent",
|
||||
"agentcore",
|
||||
"async_invoke",
|
||||
"openai",
|
||||
"mantle",
|
||||
"chat_completions",
|
||||
]:
|
||||
request_params: Mapping[str, object] | None = None,
|
||||
) -> BedrockRoute:
|
||||
"""
|
||||
Get the bedrock route for the given model.
|
||||
|
||||
``request_params`` (the caller's chat params) lets a runtime Chat Completions
|
||||
model fall back to Converse for the requests only Converse can serve.
|
||||
"""
|
||||
route_mappings: dict[
|
||||
str,
|
||||
|
|
@ -1187,7 +1230,9 @@ class BedrockModelInfo(BaseLLMModelInfo):
|
|||
if is_bedrock_application_inference_profile_arn(model):
|
||||
return "converse"
|
||||
|
||||
if uses_bedrock_runtime_chat_completions(model):
|
||||
if uses_bedrock_runtime_chat_completions(model) and not (
|
||||
request_params is not None and bedrock_request_needs_converse(model, request_params)
|
||||
):
|
||||
return "chat_completions"
|
||||
|
||||
base_model: Final = BedrockModelInfo.get_base_model(model)
|
||||
|
|
|
|||
|
|
@ -4078,7 +4078,7 @@ def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes
|
|||
if "aws_region_name" not in optional_params or optional_params["aws_region_name"] is None:
|
||||
optional_params["aws_region_name"] = aws_bedrock_client.meta.region_name
|
||||
|
||||
bedrock_route: Final = BedrockModelInfo.get_bedrock_route(model)
|
||||
bedrock_route: Final = BedrockModelInfo.get_bedrock_route(model, optional_params)
|
||||
if bedrock_route == "claude_platform":
|
||||
provider_config = ProviderConfigManager.get_provider_chat_config(
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -40870,6 +40870,7 @@
|
|||
"output_cost_per_token": 0.0
|
||||
},
|
||||
"openai.gpt-oss-120b-1:0": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -40883,6 +40884,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"openai.gpt-oss-20b-1:0": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"input_cost_per_token": 7e-08,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -58441,6 +58443,8 @@
|
|||
"supports_web_search": true
|
||||
},
|
||||
"us.openai.gpt-5.6-sol": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"input_cost_per_token": 4.4e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 8.8e-06,
|
||||
"cache_creation_input_token_cost": 5.5e-06,
|
||||
|
|
@ -58470,6 +58474,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"global.openai.gpt-5.6-sol": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"input_cost_per_token": 4e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 8e-06,
|
||||
"cache_creation_input_token_cost": 5e-06,
|
||||
|
|
@ -58499,6 +58505,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"us.openai.gpt-5.6-terra": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"input_cost_per_token": 2.2e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 4.4e-06,
|
||||
"cache_creation_input_token_cost": 2.75e-06,
|
||||
|
|
@ -58528,6 +58536,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"global.openai.gpt-5.6-terra": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 4e-06,
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
|
|
@ -58557,6 +58567,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"us.openai.gpt-5.6-luna": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"input_cost_per_token": 2.2e-07,
|
||||
"input_cost_per_token_above_272k_tokens": 4.4e-07,
|
||||
"cache_creation_input_token_cost": 2.75e-07,
|
||||
|
|
@ -58586,6 +58598,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"global.openai.gpt-5.6-luna": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"input_cost_per_token_above_272k_tokens": 4e-07,
|
||||
"cache_creation_input_token_cost": 2.5e-07,
|
||||
|
|
|
|||
|
|
@ -401,7 +401,7 @@ if TYPE_CHECKING:
|
|||
BaseVectorStoreFilesConfig,
|
||||
)
|
||||
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
|
||||
from litellm.llms.bedrock.common_utils import BedrockModelInfo
|
||||
from litellm.llms.bedrock.common_utils import BedrockModelInfo, BedrockRoute
|
||||
from litellm.llms.bedrock.embed.amazon_nova_transformation import (
|
||||
AmazonNovaEmbeddingConfig,
|
||||
)
|
||||
|
|
@ -3350,6 +3350,17 @@ def _should_drop_param(k, additional_drop_params) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def _bedrock_route_for_request(
|
||||
model: str, passed_params: Mapping[str, object], additional_drop_params: list | None
|
||||
) -> BedrockRoute:
|
||||
from litellm.llms.bedrock.common_utils import BedrockModelInfo
|
||||
|
||||
return BedrockModelInfo.get_bedrock_route(
|
||||
model,
|
||||
{k: v for k, v in passed_params.items() if not _should_drop_param(k, additional_drop_params)},
|
||||
)
|
||||
|
||||
|
||||
def _get_non_default_params(passed_params: dict, default_params: dict, additional_drop_params: list | None) -> dict:
|
||||
non_default_params: Final = {}
|
||||
for k, v in passed_params.items():
|
||||
|
|
@ -4401,9 +4412,17 @@ def get_optional_params(
|
|||
message=f"{custom_llm_provider} does not support parameters: {list(unsupported_params.keys())}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n. \n If you want to use these params dynamically send allowed_openai_params={list(unsupported_params.keys())} in your request.",
|
||||
)
|
||||
|
||||
bedrock_route: Final = (
|
||||
_bedrock_route_for_request(model, passed_params, additional_drop_params)
|
||||
if custom_llm_provider == "bedrock"
|
||||
else None
|
||||
)
|
||||
get_supported_openai_params: Final = getattr(sys.modules[__name__], "get_supported_openai_params")
|
||||
supported_params = get_supported_openai_params(
|
||||
model=model, custom_llm_provider=custom_llm_provider, base_model=base_model
|
||||
supported_params = (
|
||||
litellm.AmazonConverseConfig().get_supported_openai_params(model=model)
|
||||
if bedrock_route == "converse"
|
||||
and isinstance(provider_config, litellm.AmazonBedrockRuntimeChatCompletionsConfig)
|
||||
else get_supported_openai_params(model=model, custom_llm_provider=custom_llm_provider, base_model=base_model)
|
||||
)
|
||||
if supported_params is None:
|
||||
supported_params = get_supported_openai_params(model=model, custom_llm_provider="openai")
|
||||
|
|
@ -4573,7 +4592,6 @@ def get_optional_params(
|
|||
)
|
||||
elif custom_llm_provider == "bedrock":
|
||||
BedrockModelInfo: Final = getattr(sys.modules[__name__], "BedrockModelInfo")
|
||||
bedrock_route: Final = BedrockModelInfo.get_bedrock_route(model)
|
||||
bedrock_base_model: Final = BedrockModelInfo.get_base_model(model)
|
||||
if bedrock_route == "converse" or bedrock_route == "converse_like":
|
||||
optional_params = litellm.AmazonConverseConfig().map_openai_params(
|
||||
|
|
|
|||
|
|
@ -40870,6 +40870,7 @@
|
|||
"output_cost_per_token": 0.0
|
||||
},
|
||||
"openai.gpt-oss-120b-1:0": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -40883,6 +40884,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"openai.gpt-oss-20b-1:0": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"input_cost_per_token": 7e-08,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -58441,6 +58443,8 @@
|
|||
"supports_web_search": true
|
||||
},
|
||||
"us.openai.gpt-5.6-sol": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"input_cost_per_token": 4.4e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 8.8e-06,
|
||||
"cache_creation_input_token_cost": 5.5e-06,
|
||||
|
|
@ -58470,6 +58474,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"global.openai.gpt-5.6-sol": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"input_cost_per_token": 4e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 8e-06,
|
||||
"cache_creation_input_token_cost": 5e-06,
|
||||
|
|
@ -58499,6 +58505,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"us.openai.gpt-5.6-terra": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"input_cost_per_token": 2.2e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 4.4e-06,
|
||||
"cache_creation_input_token_cost": 2.75e-06,
|
||||
|
|
@ -58528,6 +58536,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"global.openai.gpt-5.6-terra": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 4e-06,
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
|
|
@ -58557,6 +58567,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"us.openai.gpt-5.6-luna": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"input_cost_per_token": 2.2e-07,
|
||||
"input_cost_per_token_above_272k_tokens": 4.4e-07,
|
||||
"cache_creation_input_token_cost": 2.75e-07,
|
||||
|
|
@ -58586,6 +58598,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"global.openai.gpt-5.6-luna": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"input_cost_per_token_above_272k_tokens": 4e-07,
|
||||
"cache_creation_input_token_cost": 2.5e-07,
|
||||
|
|
|
|||
|
|
@ -74,6 +74,9 @@
|
|||
"xhigh"
|
||||
]
|
||||
},
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"cache_creation_input_audio_token_cost": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
"""Native Bedrock Runtime Chat Completions: Grok stays on /openai/v1/chat/completions."""
|
||||
"""Native Bedrock Runtime Chat Completions: Grok, gpt-oss and GPT-5.6 stay on /openai/v1/chat/completions."""
|
||||
|
||||
import json
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
|
@ -9,25 +8,28 @@ import pytest
|
|||
import litellm
|
||||
from litellm.llms.bedrock.chat.chat_completions.transformation import (
|
||||
AmazonBedrockRuntimeChatCompletionsConfig,
|
||||
BedrockRuntimeChatCompletionsStreamingHandler,
|
||||
ReasoningTagSplitter,
|
||||
split_reasoning_tag,
|
||||
with_max_completion_tokens,
|
||||
)
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
BEDROCK_CONVERSE_ONLY_REQUEST_KEYS,
|
||||
BedrockModelInfo,
|
||||
bedrock_request_needs_converse,
|
||||
get_bedrock_chat_config,
|
||||
uses_bedrock_runtime_chat_completions,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def local_cost_map(monkeypatch):
|
||||
original_model_cost = litellm.model_cost
|
||||
try:
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true")
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
litellm.get_model_info.cache_clear()
|
||||
yield
|
||||
finally:
|
||||
litellm.model_cost = original_model_cost
|
||||
litellm.get_model_info.cache_clear()
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
litellm.get_model_info.cache_clear()
|
||||
yield
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -103,7 +105,27 @@ def test_transform_request_is_openai_chat_body_not_converse():
|
|||
assert "messages" in body
|
||||
|
||||
|
||||
def test_completion_posts_runtime_chat_completions(local_cost_map, monkeypatch):
|
||||
def _chat_completion_json(content, model, tool_calls=None):
|
||||
message = {"role": "assistant", "content": content, **({"tool_calls": tool_calls} if tool_calls else {})}
|
||||
return {
|
||||
"id": "chatcmpl-test",
|
||||
"object": "chat.completion",
|
||||
"created": 1733529600,
|
||||
"model": model,
|
||||
"choices": [{"index": 0, "message": message, "finish_reason": "tool_calls" if tool_calls else "stop"}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
|
||||
|
||||
CONVERSE_JSON = {
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_aws_env(monkeypatch):
|
||||
monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
|
||||
monkeypatch.delenv("AWS_BEDROCK_RUNTIME_ENDPOINT", raising=False)
|
||||
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
|
||||
|
|
@ -111,40 +133,490 @@ def test_completion_posts_runtime_chat_completions(local_cost_map, monkeypatch):
|
|||
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing")
|
||||
monkeypatch.setenv("AWS_SESSION_TOKEN", "testing")
|
||||
|
||||
requests: list[dict] = []
|
||||
|
||||
def mock_post(self, url, data=None, json=None, headers=None, **kwargs):
|
||||
requests.append({"url": url, "data": data, "json": json, "headers": headers or {}})
|
||||
return httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "chatcmpl-test",
|
||||
"object": "chat.completion",
|
||||
"created": 1733529600,
|
||||
"model": "us.xai.grok-4.6",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "ok"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
request=httpx.Request("POST", url),
|
||||
)
|
||||
def _recording_client(**response_kwargs):
|
||||
requests: list[httpx.Request] = []
|
||||
|
||||
with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post):
|
||||
response = litellm.completion(
|
||||
model="us.xai.grok-4.6",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
)
|
||||
def handle(request):
|
||||
requests.append(request)
|
||||
return httpx.Response(200, **response_kwargs)
|
||||
|
||||
return requests, HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle)))
|
||||
|
||||
|
||||
def test_completion_posts_runtime_chat_completions(local_cost_map, fake_aws_env):
|
||||
requests, client = _recording_client(json=_chat_completion_json("ok", "us.xai.grok-4.6"))
|
||||
response = litellm.completion(
|
||||
model="us.xai.grok-4.6",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "ok"
|
||||
assert len(requests) == 1
|
||||
assert requests[0]["url"] == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions"
|
||||
raw = requests[0]["data"]
|
||||
body = json.loads(raw) if isinstance(raw, (str, bytes, bytearray)) else (requests[0]["json"] or {})
|
||||
assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions"
|
||||
body = json.loads(requests[0].content)
|
||||
assert body["model"] == "us.xai.grok-4.6"
|
||||
assert body["messages"] == [{"role": "user", "content": "hello"}]
|
||||
assert "inferenceConfig" not in body
|
||||
|
||||
|
||||
OPENAI_RUNTIME_MODELS = (
|
||||
"openai.gpt-oss-20b-1:0",
|
||||
"openai.gpt-oss-120b-1:0",
|
||||
"us.openai.gpt-5.6-sol",
|
||||
"global.openai.gpt-5.6-sol",
|
||||
"us.openai.gpt-5.6-terra",
|
||||
"global.openai.gpt-5.6-terra",
|
||||
"us.openai.gpt-5.6-luna",
|
||||
"global.openai.gpt-5.6-luna",
|
||||
)
|
||||
GET_WEATHER_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [*OPENAI_RUNTIME_MODELS, "bedrock/openai.gpt-oss-20b-1:0"])
|
||||
def test_openai_runtime_models_use_chat_completions_route(local_cost_map, model):
|
||||
assert uses_bedrock_runtime_chat_completions(model) is True
|
||||
assert BedrockModelInfo.get_bedrock_route(model) == "chat_completions"
|
||||
assert isinstance(get_bedrock_chat_config(model), AmazonBedrockRuntimeChatCompletionsConfig)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["us.amazon.nova-micro-v1:0", "us.anthropic.claude-haiku-4-5-20251001-v1:0"])
|
||||
def test_nova_and_claude_stay_on_converse(local_cost_map, model):
|
||||
assert uses_bedrock_runtime_chat_completions(model) is False
|
||||
assert BedrockModelInfo.get_bedrock_route(model, {"tools": [GET_WEATHER_TOOL]}) == "converse"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["openai.gpt-oss-20b-1:0", "global.openai.gpt-5.6-sol"])
|
||||
def test_guardrail_config_falls_back_to_converse(local_cost_map, model):
|
||||
guardrail = {"guardrailIdentifier": "gr-1", "guardrailVersion": "1"}
|
||||
assert bedrock_request_needs_converse(model, {"guardrailConfig": guardrail}) is True
|
||||
assert BedrockModelInfo.get_bedrock_route(model, {"guardrailConfig": guardrail}) == "converse"
|
||||
assert BedrockModelInfo.get_bedrock_route(model, {"guardrailConfig": None}) == "chat_completions"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_params, expected_route",
|
||||
[
|
||||
({"tools": [GET_WEATHER_TOOL]}, "converse"),
|
||||
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}, "converse"),
|
||||
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": None}, "converse"),
|
||||
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "none"}, "chat_completions"),
|
||||
({"reasoning_effort": "low"}, "chat_completions"),
|
||||
({"tools": None, "reasoning_effort": "low"}, "chat_completions"),
|
||||
({"tools": [], "reasoning_effort": "low"}, "chat_completions"),
|
||||
({}, "chat_completions"),
|
||||
],
|
||||
)
|
||||
def test_gpt56_tools_need_reasoning_none_on_chat_completions(local_cost_map, request_params, expected_route):
|
||||
assert BedrockModelInfo.get_bedrock_route("global.openai.gpt-5.6-sol", request_params) == expected_route
|
||||
assert BedrockModelInfo.get_bedrock_route("bedrock/us.openai.gpt-5.6-terra", request_params) == expected_route
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reasoning_effort", ["low", "high", None])
|
||||
def test_gpt_oss_tools_with_any_reasoning_effort_stay_on_chat_completions(local_cost_map, reasoning_effort):
|
||||
params = {"tools": [GET_WEATHER_TOOL], "reasoning_effort": reasoning_effort}
|
||||
assert bedrock_request_needs_converse("openai.gpt-oss-120b-1:0", params) is False
|
||||
assert BedrockModelInfo.get_bedrock_route("openai.gpt-oss-120b-1:0", params) == "chat_completions"
|
||||
|
||||
|
||||
def test_explicit_converse_prefix_wins_for_openai_models(local_cost_map):
|
||||
assert BedrockModelInfo.get_bedrock_route("bedrock/converse/openai.gpt-oss-20b-1:0") == "converse"
|
||||
assert BedrockModelInfo.get_bedrock_route("converse/global.openai.gpt-5.6-sol", {}) == "converse"
|
||||
|
||||
|
||||
def test_map_openai_params_sends_max_tokens_as_max_completion_tokens():
|
||||
cfg = AmazonBedrockRuntimeChatCompletionsConfig()
|
||||
mapped = cfg.map_openai_params(
|
||||
non_default_params={"max_tokens": 64, "temperature": 0.1},
|
||||
optional_params={},
|
||||
model="global.openai.gpt-5.6-sol",
|
||||
drop_params=False,
|
||||
)
|
||||
assert mapped == {"max_completion_tokens": 64, "temperature": 0.1}
|
||||
|
||||
|
||||
def test_map_openai_params_keeps_explicit_max_completion_tokens():
|
||||
cfg = AmazonBedrockRuntimeChatCompletionsConfig()
|
||||
mapped = cfg.map_openai_params(
|
||||
non_default_params={"max_tokens": 64, "max_completion_tokens": 32},
|
||||
optional_params={},
|
||||
model="openai.gpt-oss-20b-1:0",
|
||||
drop_params=False,
|
||||
)
|
||||
assert mapped == {"max_completion_tokens": 32}
|
||||
|
||||
|
||||
def test_with_max_completion_tokens_leaves_other_params_alone():
|
||||
assert with_max_completion_tokens({"temperature": 0.5}) == {"temperature": 0.5}
|
||||
|
||||
|
||||
def test_supported_params_include_reasoning_effort_for_gpt56(local_cost_map):
|
||||
cfg = AmazonBedrockRuntimeChatCompletionsConfig()
|
||||
assert "reasoning_effort" in cfg.get_supported_openai_params("global.openai.gpt-5.6-sol")
|
||||
assert "reasoning_effort" in cfg.get_supported_openai_params("openai.gpt-oss-20b-1:0")
|
||||
|
||||
|
||||
def test_split_reasoning_tag_splits_leading_tag():
|
||||
assert split_reasoning_tag("<reasoning>plan it\n</reasoning>\n\nHello") == ("plan it\n", "Hello")
|
||||
|
||||
|
||||
def test_split_reasoning_tag_drops_an_empty_tag():
|
||||
assert split_reasoning_tag("<reasoning></reasoning>Hello") == (None, "Hello")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"content",
|
||||
[
|
||||
"<reasoning>plan it\n</reasoning>\n\nHello",
|
||||
"<reasoning>never closed",
|
||||
"<reas",
|
||||
"Hello <reasoning>later</reasoning>",
|
||||
"<reasoning></reasoning>",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("chunk_size", [1, 3, 7])
|
||||
def test_split_reasoning_tag_matches_the_streamed_split(content, chunk_size):
|
||||
chunks = [content[start : start + chunk_size] for start in range(0, len(content), chunk_size)]
|
||||
streamed_reasoning, streamed_content = _run_splitter(chunks)
|
||||
|
||||
assert split_reasoning_tag(content) == (streamed_reasoning or None, streamed_content)
|
||||
|
||||
|
||||
def test_split_reasoning_tag_passes_plain_content_through():
|
||||
assert split_reasoning_tag("Hello") == (None, "Hello")
|
||||
|
||||
|
||||
def test_split_reasoning_tag_ignores_tag_after_content_starts():
|
||||
content = "Hello <reasoning>not mine</reasoning>"
|
||||
assert split_reasoning_tag(content) == (None, content)
|
||||
|
||||
|
||||
def _run_splitter(chunks):
|
||||
state = ReasoningTagSplitter()
|
||||
reasoning = ""
|
||||
content = ""
|
||||
for chunk in chunks:
|
||||
state, fed_reasoning, fed_content = state.feed(chunk)
|
||||
reasoning += fed_reasoning
|
||||
content += fed_content
|
||||
state, flushed_reasoning, flushed_content = state.flush()
|
||||
return reasoning + flushed_reasoning, content + flushed_content
|
||||
|
||||
|
||||
def test_reasoning_tag_splitter_handles_tags_split_across_chunks():
|
||||
assert _run_splitter(["<reas", "oning>I think", " so</reas", "oning>\n\nHel", "lo"]) == ("I think so", "Hello")
|
||||
|
||||
|
||||
def test_reasoning_tag_splitter_passes_plain_content_through():
|
||||
assert _run_splitter(["Hel", "lo <reasoning>later</reasoning>"]) == ("", "Hello <reasoning>later</reasoning>")
|
||||
|
||||
|
||||
def test_reasoning_tag_splitter_flushes_unclosed_reasoning():
|
||||
assert _run_splitter(["<reasoning>never clo", "sed"]) == ("never closed", "")
|
||||
|
||||
|
||||
def test_reasoning_tag_splitter_releases_a_false_tag_prefix():
|
||||
assert _run_splitter(["<", "b>x"]) == ("", "<b>x")
|
||||
|
||||
|
||||
def _stream_chunk(delta, finish_reason=None, index=0):
|
||||
return {
|
||||
"id": "chatcmpl-test",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1733529600,
|
||||
"model": "openai.gpt-oss-20b-1:0",
|
||||
"choices": [{"index": index, "delta": delta, "finish_reason": finish_reason}],
|
||||
}
|
||||
|
||||
|
||||
def test_streaming_handler_splits_reasoning_deltas_per_choice():
|
||||
handler = BedrockRuntimeChatCompletionsStreamingHandler(streaming_response=iter(()), sync_stream=True)
|
||||
|
||||
first = handler.chunk_parser(_stream_chunk({"role": "assistant", "content": "<reasoning>I think"}))
|
||||
assert first.choices[0].delta.reasoning_content == "I think"
|
||||
assert not first.choices[0].delta.content
|
||||
|
||||
second = handler.chunk_parser(_stream_chunk({"content": " so</reasoning>\n\nHello"}))
|
||||
assert second.choices[0].delta.reasoning_content == " so"
|
||||
assert second.choices[0].delta.content == "Hello"
|
||||
|
||||
tool_call = {"index": 0, "id": "call_0", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}
|
||||
third = handler.chunk_parser(_stream_chunk({"content": None, "tool_calls": [tool_call]}))
|
||||
assert third.choices[0].delta.tool_calls[0].function.name == "get_weather"
|
||||
|
||||
last = handler.chunk_parser(_stream_chunk({}, finish_reason="stop"))
|
||||
assert last.choices[0].finish_reason == "stop"
|
||||
|
||||
|
||||
def _reasoning_of(parsed):
|
||||
return getattr(parsed.choices[0].delta, "reasoning_content", None)
|
||||
|
||||
|
||||
def test_streaming_handler_keeps_split_state_per_choice_index():
|
||||
handler = BedrockRuntimeChatCompletionsStreamingHandler(streaming_response=iter(()), sync_stream=True)
|
||||
|
||||
opened = handler.chunk_parser(_stream_chunk({"content": "<reasoning>first"}, index=0))
|
||||
assert _reasoning_of(opened) == "first"
|
||||
|
||||
plain = handler.chunk_parser(_stream_chunk({"content": "plain answer"}, index=1))
|
||||
assert _reasoning_of(plain) is None
|
||||
assert plain.choices[0].delta.content == "plain answer"
|
||||
|
||||
still_reasoning = handler.chunk_parser(_stream_chunk({"content": " more"}, index=0))
|
||||
assert _reasoning_of(still_reasoning) == " more"
|
||||
assert not still_reasoning.choices[0].delta.content
|
||||
|
||||
|
||||
def test_streaming_handler_flushes_held_text_on_an_empty_final_delta():
|
||||
handler = BedrockRuntimeChatCompletionsStreamingHandler(streaming_response=iter(()), sync_stream=True)
|
||||
|
||||
held = handler.chunk_parser(_stream_chunk({"content": "<reas"}))
|
||||
assert not held.choices[0].delta.content
|
||||
|
||||
final = handler.chunk_parser(_stream_chunk({}, finish_reason="stop"))
|
||||
assert final.choices[0].delta.content == "<reas"
|
||||
assert _reasoning_of(final) is None
|
||||
|
||||
unclosed = BedrockRuntimeChatCompletionsStreamingHandler(streaming_response=iter(()), sync_stream=True)
|
||||
unclosed.chunk_parser(_stream_chunk({"content": "<reasoning>almost done</reas"}))
|
||||
drained = unclosed.chunk_parser(_stream_chunk({}, finish_reason="length"))
|
||||
assert _reasoning_of(drained) == "</reas"
|
||||
|
||||
|
||||
def test_gpt_oss_completion_hits_chat_completions_and_splits_reasoning(local_cost_map, fake_aws_env):
|
||||
requests, client = _recording_client(
|
||||
json=_chat_completion_json("<reasoning>plan</reasoning>\n\nHi", "openai.gpt-oss-20b-1:0")
|
||||
)
|
||||
response = litellm.completion(
|
||||
model="bedrock/openai.gpt-oss-20b-1:0",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
max_tokens=64,
|
||||
reasoning_effort="low",
|
||||
tools=[GET_WEATHER_TOOL],
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions"
|
||||
body = json.loads(requests[0].content)
|
||||
assert body["model"] == "openai.gpt-oss-20b-1:0"
|
||||
assert body["max_completion_tokens"] == 64
|
||||
assert "max_tokens" not in body
|
||||
assert body["reasoning_effort"] == "low"
|
||||
assert body["tools"] == [GET_WEATHER_TOOL]
|
||||
assert response.choices[0].message.reasoning_content == "plan"
|
||||
assert response.choices[0].message.content == "Hi"
|
||||
|
||||
|
||||
def test_gpt56_tools_with_reasoning_effort_go_to_converse(local_cost_map, fake_aws_env):
|
||||
requests, client = _recording_client(json=CONVERSE_JSON)
|
||||
response = litellm.completion(
|
||||
model="bedrock/global.openai.gpt-5.6-sol",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
tools=[GET_WEATHER_TOOL],
|
||||
reasoning_effort="low",
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert requests[0].url.raw_path.endswith(b"/model/global.openai.gpt-5.6-sol/converse")
|
||||
assert json.loads(requests[0].content)["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather"
|
||||
assert response.choices[0].message.content == "ok"
|
||||
|
||||
|
||||
def test_gpt56_tools_with_reasoning_none_stay_on_chat_completions(local_cost_map, fake_aws_env):
|
||||
tool_calls = [
|
||||
{"id": "call_0", "type": "function", "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}}
|
||||
]
|
||||
requests, client = _recording_client(json=_chat_completion_json(None, "global.openai.gpt-5.6-sol", tool_calls))
|
||||
response = litellm.completion(
|
||||
model="bedrock/global.openai.gpt-5.6-sol",
|
||||
messages=[{"role": "user", "content": "weather in Paris"}],
|
||||
tools=[GET_WEATHER_TOOL],
|
||||
reasoning_effort="none",
|
||||
max_tokens=64,
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions"
|
||||
body = json.loads(requests[0].content)
|
||||
assert body["tools"] == [GET_WEATHER_TOOL]
|
||||
assert body["reasoning_effort"] == "none"
|
||||
assert body["max_completion_tokens"] == 64
|
||||
assert response.choices[0].message.tool_calls[0].function.name == "get_weather"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"converse_only_param",
|
||||
[
|
||||
{"guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "1"}},
|
||||
{"performanceConfig": {"latency": "optimized"}},
|
||||
{"requestMetadata": {"team": "search"}},
|
||||
{"serviceTier": {"type": "priority"}},
|
||||
],
|
||||
ids=lambda param: next(iter(param)),
|
||||
)
|
||||
def test_converse_only_request_keys_go_to_converse(local_cost_map, fake_aws_env, converse_only_param):
|
||||
requests, client = _recording_client(json=CONVERSE_JSON)
|
||||
litellm.completion(
|
||||
model="bedrock/openai.gpt-oss-20b-1:0",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
client=client,
|
||||
**converse_only_param,
|
||||
)
|
||||
|
||||
assert requests[0].url.raw_path.endswith(b"/model/openai.gpt-oss-20b-1%3A0/converse")
|
||||
(key, value), = converse_only_param.items()
|
||||
assert json.loads(requests[0].content)[key] == value
|
||||
|
||||
|
||||
def test_converse_only_keys_cover_every_converse_config_block():
|
||||
assert set(litellm.AmazonConverseConfig.get_config_blocks()) <= BEDROCK_CONVERSE_ONLY_REQUEST_KEYS
|
||||
|
||||
|
||||
def test_operator_owned_request_metadata_goes_to_converse(local_cost_map, fake_aws_env, monkeypatch):
|
||||
monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ["user_api_key_team_alias"])
|
||||
requests, client = _recording_client(json=CONVERSE_JSON)
|
||||
litellm.completion(
|
||||
model="bedrock/openai.gpt-oss-20b-1:0",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
metadata={"user_api_key_team_alias": "search"},
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert requests[0].url.raw_path.endswith(b"/model/openai.gpt-oss-20b-1%3A0/converse")
|
||||
assert json.loads(requests[0].content)["requestMetadata"] == {"user_api_key_team_alias": "search"}
|
||||
|
||||
|
||||
def test_dropped_converse_only_key_keeps_the_request_on_chat_completions(local_cost_map, fake_aws_env):
|
||||
requests, client = _recording_client(json=_chat_completion_json("ok", "openai.gpt-oss-20b-1:0"))
|
||||
litellm.completion(
|
||||
model="bedrock/openai.gpt-oss-20b-1:0",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
guardrailConfig={"guardrailIdentifier": "gr-1", "guardrailVersion": "1"},
|
||||
additional_drop_params=["guardrailConfig"],
|
||||
max_tokens=8,
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions"
|
||||
body = json.loads(requests[0].content)
|
||||
assert "guardrailConfig" not in body
|
||||
assert body["max_completion_tokens"] == 8
|
||||
assert "inferenceConfig" not in body
|
||||
|
||||
|
||||
def test_dropped_tools_keep_gpt56_reasoning_request_on_chat_completions(local_cost_map, fake_aws_env):
|
||||
requests, client = _recording_client(json=_chat_completion_json("ok", "global.openai.gpt-5.6-sol"))
|
||||
litellm.completion(
|
||||
model="bedrock/global.openai.gpt-5.6-sol",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
tools=[GET_WEATHER_TOOL],
|
||||
reasoning_effort="low",
|
||||
additional_drop_params=["tools"],
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions"
|
||||
body = json.loads(requests[0].content)
|
||||
assert "tools" not in body
|
||||
assert body["reasoning_effort"] == "low"
|
||||
|
||||
|
||||
def test_legacy_functions_stay_on_chat_completions(local_cost_map, fake_aws_env):
|
||||
requests, client = _recording_client(json=_chat_completion_json("ok", "openai.gpt-oss-20b-1:0"))
|
||||
litellm.completion(
|
||||
model="bedrock/openai.gpt-oss-20b-1:0",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
functions=[GET_WEATHER_TOOL["function"]],
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions"
|
||||
assert json.loads(requests[0].content)["functions"] == [GET_WEATHER_TOOL["function"]]
|
||||
|
||||
|
||||
def test_converse_fallback_validates_against_converse_params(local_cost_map, fake_aws_env):
|
||||
requests, client = _recording_client(json=CONVERSE_JSON)
|
||||
guardrail = {"guardrailIdentifier": "gr-1", "guardrailVersion": "1"}
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match="seed"):
|
||||
litellm.completion(
|
||||
model="bedrock/openai.gpt-oss-20b-1:0",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
guardrailConfig=guardrail,
|
||||
seed=7,
|
||||
client=client,
|
||||
)
|
||||
litellm.completion(
|
||||
model="bedrock/openai.gpt-oss-20b-1:0",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
guardrailConfig=guardrail,
|
||||
seed=7,
|
||||
drop_params=True,
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert requests[0].url.raw_path.endswith(b"/model/openai.gpt-oss-20b-1%3A0/converse")
|
||||
assert "seed" not in json.loads(requests[0].content)
|
||||
|
||||
|
||||
def test_n_is_rejected_before_reaching_chat_completions(local_cost_map, fake_aws_env):
|
||||
requests, client = _recording_client(json=_chat_completion_json("ok", "openai.gpt-oss-20b-1:0"))
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match="'n'"):
|
||||
litellm.completion(
|
||||
model="bedrock/openai.gpt-oss-20b-1:0",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
n=2,
|
||||
client=client,
|
||||
)
|
||||
litellm.completion(
|
||||
model="bedrock/openai.gpt-oss-20b-1:0",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
n=2,
|
||||
drop_params=True,
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert "n" not in json.loads(requests[0].content)
|
||||
|
||||
|
||||
def _sse(chunks):
|
||||
return ("".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n").encode()
|
||||
|
||||
|
||||
def test_gpt_oss_streaming_completion_splits_reasoning(local_cost_map, fake_aws_env):
|
||||
chunks = (
|
||||
_stream_chunk({"role": "assistant", "content": "<reasoning>plan"}),
|
||||
_stream_chunk({"content": "</reasoning>\n\nHi"}),
|
||||
_stream_chunk({}, finish_reason="stop"),
|
||||
)
|
||||
requests, client = _recording_client(content=_sse(chunks), headers={"content-type": "text/event-stream"})
|
||||
stream = litellm.completion(
|
||||
model="bedrock/openai.gpt-oss-20b-1:0",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
stream=True,
|
||||
client=client,
|
||||
)
|
||||
deltas = [chunk.choices[0].delta for chunk in stream]
|
||||
|
||||
assert [str(request.url) for request in requests] == [
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions"
|
||||
]
|
||||
assert json.loads(requests[0].content)["stream"] is True
|
||||
assert "".join(getattr(delta, "reasoning_content", None) or "" for delta in deltas) == "plan"
|
||||
assert "".join(delta.content or "" for delta in deltas) == "Hi"
|
||||
|
||||
|
||||
def test_streaming_handler_keeps_native_reasoning_next_to_the_tagged_split():
|
||||
handler = BedrockRuntimeChatCompletionsStreamingHandler(streaming_response=iter(()), sync_stream=True)
|
||||
parsed = handler.chunk_parser(
|
||||
_stream_chunk({"reasoning": "native ", "content": "<reasoning>tagged</reasoning>Hi"}, finish_reason="stop")
|
||||
)
|
||||
|
||||
assert parsed.choices[0].delta.reasoning_content == "native tagged"
|
||||
assert parsed.choices[0].delta.content == "Hi"
|
||||
|
|
|
|||
|
|
@ -138,9 +138,12 @@ def _bedrock_response(model, usage):
|
|||
|
||||
|
||||
@pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id)
|
||||
def test_bedrock_gpt_5_6_profiles_route_to_converse(profile, local_model_cost_map):
|
||||
"""GPT-5.6 is served by Converse on bedrock-runtime, never by Invoke."""
|
||||
assert BedrockModelInfo.get_bedrock_route(f"bedrock/{profile.model_id}") == "converse"
|
||||
def test_bedrock_gpt_5_6_profiles_never_route_to_invoke(profile, local_model_cost_map):
|
||||
"""GPT-5.6 is served by bedrock-runtime's native Chat Completions, and by Converse when
|
||||
the request carries function tools without reasoning_effort "none", never by Invoke."""
|
||||
assert BedrockModelInfo.get_bedrock_route(f"bedrock/{profile.model_id}") == "chat_completions"
|
||||
tools_with_reasoning = {"tools": [{"type": "function", "function": {"name": "f"}}], "reasoning_effort": "low"}
|
||||
assert BedrockModelInfo.get_bedrock_route(f"bedrock/{profile.model_id}", tools_with_reasoning) == "converse"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id)
|
||||
|
|
|
|||
|
|
@ -878,6 +878,8 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
|
|||
"supports_video_input": {"type": "boolean"},
|
||||
"supports_vision": {"type": "boolean"},
|
||||
"supports_web_search": {"type": "boolean"},
|
||||
"use_bedrock_runtime_chat_completions": {"type": "boolean"},
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": {"type": "boolean"},
|
||||
"supports_url_context": {"type": "boolean"},
|
||||
"supports_multimodal": {"type": "boolean"},
|
||||
"uses_embed_content": {"type": "boolean"},
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue