scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

Merge 6daf0b7136 into d729f975aa

Browse source
This commit is contained in:
Mateo Wang 2026-10-02 02:00:27 +00:00 • committed by GitHub
commit c56fcbeb01
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 4571 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
litellm-rust/crates/model-catalog/src/model_info.rs
View file

@ -467,6 +467,10 @@ pub struct ModelInfo {
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_audio_output: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_bedrock_runtime_chat_completions_response_format: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_bedrock_runtime_chat_completions_tools_with_reasoning: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_computer_use: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_embedding_image_input: Option<bool>,

3
litellm/__init__.py
View file

@ -1780,6 +1780,9 @@ if TYPE_CHECKING:
from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import (
AmazonBedrockOpenAIConfig as AmazonBedrockOpenAIConfig,
)
from .llms.bedrock.chat.chat_completions.transformation import (
AmazonBedrockRuntimeChatCompletionsConfig as AmazonBedrockRuntimeChatCompletionsConfig,
)
from .llms.bedrock.image_generation.amazon_stability1_transformation import (
AmazonStabilityConfig as AmazonStabilityConfig,
)

5
litellm/_lazy_imports_registry.py
View file

@ -206,6 +206,7 @@ LLM_CONFIG_NAMES: Final = (
"AmazonTwelveLabsPegasusConfig",
"AmazonInvokeConfig",
"AmazonBedrockOpenAIConfig",
"AmazonBedrockRuntimeChatCompletionsConfig",
"AmazonStabilityConfig",
"AmazonStability3Config",
"AmazonNovaCanvasConfig",
@ -868,6 +869,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
".llms.bedrock.chat.invoke_transformations.amazon_openai_transformation",
"AmazonBedrockOpenAIConfig",
),
"AmazonBedrockRuntimeChatCompletionsConfig": (
".llms.bedrock.chat.chat_completions.transformation",
"AmazonBedrockRuntimeChatCompletionsConfig",
),
"AmazonStabilityConfig": (
".llms.bedrock.image_generation.amazon_stability1_transformation",
"AmazonStabilityConfig",

61
litellm/litellm_core_utils/prompt_templates/image_handling.py
View file

@ -4,8 +4,9 @@ Helper functions to handle images passed in messages
import asyncio
import base64
from collections.abc import Callable, Mapping
from collections.abc import Callable, Iterable, Mapping
from dataclasses import dataclass
from itertools import chain
from types import MappingProxyType
from typing import Final
@ -15,6 +16,7 @@ import litellm
from litellm import verbose_logger
from litellm.caching.caching import InMemoryCache
from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB
from litellm.litellm_core_utils.prompt_templates.common_utils import infer_content_type_from_url_and_content
from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get
from litellm.types.llms.openai import AllMessageValues
@ -55,23 +57,16 @@ def _process_image_response(response: Response, url: str) -> str:
base64_image: Final = base64.b64encode(image_bytes).decode("utf-8")
image_type: Final = response.headers.get("Content-Type")
if image_type is None:
img_type = url.split(".")[-1].lower()
_img_type: Final = {
"jpg": "image/jpeg",
"jpeg": "image/jpeg",
"png": "image/png",
"gif": "image/gif",
"webp": "image/webp",
}.get(img_type)
if _img_type is None:
raise Exception(
f"Error: Unsupported image format. Format={_img_type}. Supported types = ['image/jpeg', 'image/png', 'image/gif', 'image/webp']"
)
img_type = _img_type
else:
img_type = image_type
try:
img_type: Final = infer_content_type_from_url_and_content(
url=url,
content=bytes(image_bytes),
current_content_type=response.headers.get("Content-Type"),
)
except ValueError as e:
raise litellm.ImageFetchError(
f"Error: Unable to determine image content type from the server's headers, the URL, or the image bytes. url={url}"
) from e
result: Final = f"data:{img_type};base64,{base64_image}"
in_memory_cache.set_cache(url, result)
@ -308,18 +303,32 @@ async def _fetch_data_urls(remote_urls: tuple[str, ...]) -> tuple[str, ...]:
raise
def _remote_urls_to_inline(
messages: Iterable[AllMessageValues], should_inline: Callable[[RemoteMedia], bool]
) -> tuple[str, ...]:
parts: Final = chain.from_iterable(_content_parts(message) for message in messages)
remotes: Final = (remote for part in parts if (remote := _parse_remote_part(part)) is not None)
return tuple(dict.fromkeys(remote.url for remote in remotes if should_inline(_remote_media(remote))))
def inline_remote_media(
messages: list[AllMessageValues], # mutable-ok: every transform_request takes list[AllMessageValues]
should_inline: Callable[[RemoteMedia], bool] = inline_every_remote_url,
) -> list[AllMessageValues]: # mutable-ok: every transform_request takes list[AllMessageValues]
remote_urls: Final = _remote_urls_to_inline(messages, should_inline)
if not remote_urls:
return messages
data_urls: Final = MappingProxyType({url: convert_url_to_base64(url) for url in remote_urls})
return [ # mutable-ok: transform_request takes a list
_inline_message(message, data_urls, should_inline) for message in messages
]
async def async_inline_remote_media(
messages: list[AllMessageValues], # mutable-ok: every transform_request takes list[AllMessageValues]
should_inline: Callable[[RemoteMedia], bool] = inline_every_remote_url,
) -> list[AllMessageValues]: # mutable-ok: every transform_request takes list[AllMessageValues]
remote_urls: Final = tuple(
dict.fromkeys(
remote.url
for message in messages
for part in _content_parts(message)
if (remote := _parse_remote_part(part)) is not None and should_inline(_remote_media(remote))
)
)
remote_urls: Final = _remote_urls_to_inline(messages, should_inline)
if not remote_urls:
return messages
data_urls: Final = await _fetch_data_urls(remote_urls)

500
litellm/llms/bedrock/chat/chat_completions/transformation.py Normal file
View file

@ -0,0 +1,500 @@
"""
Native OpenAI Chat Completions on Amazon Bedrock Runtime.
AWS serves this surface at
``https://bedrock-runtime.{region}.amazonaws.com/openai/v1/chat/completions``
for Grok 4.6, gpt-oss and GPT 5.6 and newer. GPT 5.6 and newer take it by default
(``bedrock_runtime_chat_completions_is_default`` in ``common_utils``), so their chat
completions stay chat completions instead of being rewritten to Converse; the
``chat_completions/`` route prefix opts any other model in, and ``converse/`` pins a
model to Converse.
Usage: model="bedrock/global.openai.gpt-6-sol" or
model="bedrock/chat_completions/openai.gpt-oss-20b-1:0". A request that needs a
Converse-only feature (``bedrock_request_needs_converse`` in ``common_utils``) is
still served by Converse.
"""
from collections.abc import AsyncIterator, Iterator, Mapping
from dataclasses import dataclass, replace
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal
import httpx
from typing_extensions import assert_never
import litellm
from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_inline_remote_media,
inline_remote_image_urls,
inline_remote_media,
)
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token
from litellm.llms.bedrock.common_utils import (
BedrockError,
bedrock_model_is_openai_gpt,
split_bedrock_region_path,
)
from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler
from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices, ModelResponse, ModelResponseStream
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
REASONING_OPEN_TAG: Final = "<reasoning>"
REASONING_CLOSE_TAG: Final = "</reasoning>"
CHAT_COMPLETIONS_REFUSED_PARAMS_BY_FAMILY: Final = MappingProxyType(
{
"openai.gpt-oss": frozenset(("logit_bias",)),
"xai.": frozenset(("frequency_penalty", "presence_penalty")),
}
)
GPT_CHAT_COMPLETIONS_PARAMS_REFUSED_WHILE_REASONING: Final = frozenset(
("temperature", "top_p", "frequency_penalty", "presence_penalty", "logprobs", "top_logprobs")
)
def chat_completions_params_refused_for(model: str) -> frozenset[str]:
"""The OpenAI params AWS's Chat Completions endpoint rejects for this model whatever else the request says.
GPT-OSS answers ``logit_bias`` with a 400 and Grok answers the penalties with a 503, so the native config leaves
them out of its supported params and litellm refuses them, or drops them under ``drop_params``, before sending.
"""
model_id: Final = split_bedrock_region_path(model)[1]
return frozenset().union(
*(refused for family, refused in CHAT_COMPLETIONS_REFUSED_PARAMS_BY_FAMILY.items() if family in model_id)
)
def chat_completions_params_refused_while_reasoning(model: str, params: Mapping[str, object]) -> frozenset[str]:
"""The params of this request that AWS ties to ``reasoning_effort: "none"`` on the GPT-5.x and GPT-6.x families.
AWS answers ``temperature``, ``top_p``, the penalties, and logprobs with a 400 while the model reasons, which
is every effort but ``"none"`` and the default when none is set, and accepts all of them under ``"none"``.
"""
if params.get("reasoning_effort") == "none" or not bedrock_model_is_openai_gpt(model):
return frozenset()
return GPT_CHAT_COMPLETIONS_PARAMS_REFUSED_WHILE_REASONING & frozenset(params)
def _without_params(params: Mapping[str, object], dropped: frozenset[str]) -> Mapping[str, object]:
return MappingProxyType({key: value for key, value in params.items() if key not in dropped})
CHAT_COMPLETIONS_REFUSED_REASONING_EFFORTS_BY_FAMILY: Final = MappingProxyType({"xai.": frozenset(("none",))})
def chat_completions_reasoning_efforts_refused_for(model: str) -> frozenset[str]:
"""The ``reasoning_effort`` values AWS's Chat Completions endpoint rejects for this model.
Grok answers ``"none"`` with a 400 (it takes low, medium, high, and xhigh) where Converse dropped every
``reasoning_effort`` for it, so the native config drops the value and AWS applies its default effort as before.
"""
model_id: Final = split_bedrock_region_path(model)[1]
return frozenset().union(
*(
refused
for family, refused in CHAT_COMPLETIONS_REFUSED_REASONING_EFFORTS_BY_FAMILY.items()
if family in model_id
)
)
def without_refused_reasoning_effort(model: str, params: Mapping[str, object]) -> Mapping[str, object]:
effort: Final = params.get("reasoning_effort")
if not isinstance(effort, str) or effort not in chat_completions_reasoning_efforts_refused_for(model):
return params
return _without_params(params, frozenset(("reasoning_effort",)))
def non_string_reasoning_effort(params: Mapping[str, object]) -> frozenset[str]:
"""``reasoning_effort`` when the request sends it as anything but a string (an int, a list, an object).
AWS's Chat Completions endpoint answers such a value with a 400 where Converse silently dropped it, so the
native config refuses it before the call, or drops it under ``drop_params`` so AWS applies its default effort.
"""
effort: Final = params.get("reasoning_effort")
if effort is None or isinstance(effort, str):
return frozenset()
return frozenset(("reasoning_effort",))
def _held_close_tag_prefix(text: str) -> int:
return next(
(
size
for size in range(min(len(text), len(REASONING_CLOSE_TAG) - 1), 0, -1)
if REASONING_CLOSE_TAG.startswith(text[-size:])
),
0,
)
@dataclass(frozen=True, slots=True)
class ReasoningTagSplitter:
"""
The same split for a stream of content deltas, where a tag can arrive across chunks.
``feed`` returns the next state plus the reasoning and content text the delta contributes;
``flush`` releases what the stream ended on before a tag resolved.
"""
phase: Literal["start", "reasoning", "after_close", "content"] = "start"
pending: str = ""
def feed(self, text: str) -> tuple["ReasoningTagSplitter", str, str]:
match self.phase:
case "content":
return self, "", text
case "after_close":
content: Final = text.lstrip()
return (replace(self, phase="content") if content else self), "", content
case "start":
return self._feed_start(self.pending + text)
case "reasoning":
return self._feed_reasoning(self.pending + text)
case _:
assert_never(self.phase)
def _feed_start(self, buffered: str) -> tuple["ReasoningTagSplitter", str, str]:
if buffered.startswith(REASONING_OPEN_TAG):
return replace(self, phase="reasoning", pending="")._feed_reasoning(buffered[len(REASONING_OPEN_TAG) :])
if REASONING_OPEN_TAG.startswith(buffered):
return replace(self, pending=buffered), "", ""
return replace(self, phase="content", pending=""), "", buffered
def _feed_reasoning(self, buffered: str) -> tuple["ReasoningTagSplitter", str, str]:
close_at: Final = buffered.find(REASONING_CLOSE_TAG)
if close_at >= 0:
after_close: Final = replace(self, phase="after_close", pending="")
next_state, _, content = after_close.feed(buffered[close_at + len(REASONING_CLOSE_TAG) :])
return next_state, buffered[:close_at], content
held: Final = _held_close_tag_prefix(buffered)
return replace(self, pending=buffered[len(buffered) - held :]), buffered[: len(buffered) - held], ""
def flush(self) -> tuple["ReasoningTagSplitter", str, str]:
drained: Final = replace(self, phase="content", pending="")
if self.phase == "reasoning":
return drained, self.pending, ""
return drained, "", self.pending
def _split_streamed_content(
splitter: ReasoningTagSplitter, content: str | None, finished: bool
) -> tuple[ReasoningTagSplitter, str, str]:
fed_state, fed_reasoning, fed_content = splitter.feed(content or "")
if not finished:
return fed_state, fed_reasoning, fed_content
drained, flushed_reasoning, flushed_content = fed_state.flush()
return drained, fed_reasoning + flushed_reasoning, fed_content + flushed_content
def split_reasoning_tag(content: str) -> tuple[str | None, str]:
"""
Split gpt-oss's inline ``<reasoning>...</reasoning>`` prefix out of a complete message.
Runs the streaming splitter over the whole message, so a streamed and a non-streamed
response to the same completion split identically. Returns ``(None, content)`` when the
message does not start with the tag.
"""
_, reasoning, body = _split_streamed_content(ReasoningTagSplitter(), content, finished=True)
return reasoning or None, body
class BedrockRuntimeChatCompletionsStreamingHandler(OpenAIChatCompletionStreamingHandler):
"""OpenAI chunk parsing plus the ``<reasoning>`` split, tracked per choice index."""
def __init__(
self,
streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse,
sync_stream: bool,
json_mode: bool | None = False,
) -> None:
super().__init__(streaming_response=streaming_response, sync_stream=sync_stream, json_mode=json_mode)
self._splitters: Mapping[int, ReasoningTagSplitter] = MappingProxyType({})
def chunk_parser(self, chunk: dict) -> ModelResponseStream: # mutable-ok: BaseModelResponseIterator signature
parsed: Final = super().chunk_parser(chunk)
for choice in parsed.choices:
next_state, reasoning, content = _split_streamed_content(
self._splitters.get(choice.index, ReasoningTagSplitter()),
choice.delta.content,
choice.finish_reason is not None,
)
self._splitters = MappingProxyType({**self._splitters, choice.index: next_state})
if reasoning:
choice.delta.reasoning_content = f"{getattr(choice.delta, 'reasoning_content', None) or ''}{reasoning}"
if content or choice.delta.content is not None:
choice.delta.content = content
return parsed
def with_max_completion_tokens(params: Mapping[str, object]) -> Mapping[str, object]:
"""
Send the caller's ``max_tokens`` as ``max_completion_tokens``.
Every model on this surface accepts ``max_completion_tokens`` and the GPT-5.6 family
rejects ``max_tokens``; an explicit ``max_completion_tokens`` wins when both are set.
"""
if "max_tokens" not in params:
return params
return MappingProxyType(
{
key: value
for key, value in (("max_completion_tokens", params["max_tokens"]), *params.items())
if key != "max_tokens"
}
)
class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
def __init__(self, aws_signer: BaseAWSLLM | None = None) -> None:
super().__init__()
self._aws_signer: Final = aws_signer or BaseAWSLLM()
@property
def custom_llm_provider(self) -> str | None:
return "bedrock"
@property
def uses_async_transform_request(self) -> bool:
return True
def get_error_class(
self,
error_message: str,
status_code: int,
headers: dict[str, object] | httpx.Headers, # mutable-ok: BaseConfig signature
) -> BaseLLMException:
return BedrockError(status_code=status_code, message=error_message, headers=headers)
def validate_environment(
self,
headers: dict, # mutable-ok: BaseConfig signature
model: str,
messages: list[AllMessageValues],
optional_params: dict, # mutable-ok: BaseConfig signature
litellm_params: dict, # mutable-ok: BaseConfig signature
api_key: str | None = None,
api_base: str | None = None,
) -> dict: # mutable-ok: BaseConfig signature
return super().validate_environment(
headers=headers,
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
api_key=bedrock_bearer_token(api_key),
api_base=api_base,
)
def get_complete_url(
self,
api_base: str | None,
api_key: str | None,
model: str,
optional_params: dict, # mutable-ok: BaseConfig signature
litellm_params: dict, # mutable-ok: BaseConfig signature
stream: bool | None = None,
) -> str:
if api_base is not None and "chat/completions" in api_base:
return api_base.rstrip("/")
aws_region_name: Final = self._aws_signer._get_aws_region_name( # pyright: ignore[reportPrivateUsage] # BaseAWSLLM has no public region resolver
optional_params=self._params_with_region_from_path(optional_params, model), model=model
)
_, proxy_endpoint_url = self._aws_signer.get_runtime_endpoint(
api_base=api_base,
aws_bedrock_runtime_endpoint=optional_params.get("aws_bedrock_runtime_endpoint"),
aws_region_name=aws_region_name,
)
base: Final = proxy_endpoint_url.rstrip("/")
if base.endswith("/openai/v1/chat/completions"):
return base
if base.endswith("/openai/v1"):
return f"{base}/chat/completions"
return f"{base}/openai/v1/chat/completions"
def _params_with_region_from_path(
self, optional_params: dict, model: str | None
) -> dict: # mutable-ok: BaseAWSLLM's region resolver and signer take a plain dict
region_from_path, _ = split_bedrock_region_path(model or "")
if region_from_path is None or optional_params.get("aws_region_name") is not None:
return optional_params
return {**optional_params, "aws_region_name": region_from_path} # mutable-ok: BaseAWSLLM takes a plain dict
def sign_request(
self,
headers: dict, # mutable-ok: BaseConfig signature
optional_params: dict, # mutable-ok: BaseConfig signature
request_data: dict, # mutable-ok: BaseConfig signature
api_base: str,
api_key: str | None = None,
model: str | None = None,
stream: bool | None = None,
fake_stream: bool | None = None,
) -> tuple[dict, bytes | None]: # mutable-ok: BaseConfig signature
return self._aws_signer._sign_request( # pyright: ignore[reportPrivateUsage] # BaseAWSLLM has no public signer
service_name="bedrock",
headers=headers,
optional_params=self._params_with_region_from_path(optional_params, model),
request_data=request_data,
api_base=api_base,
api_key=api_key,
model=model,
stream=stream,
fake_stream=fake_stream,
)
def map_openai_params(
self,
non_default_params: dict, # mutable-ok: BaseConfig signature
optional_params: dict, # mutable-ok: BaseConfig signature
model: str,
drop_params: bool,
replace_max_completion_tokens_with_max_tokens: bool = False,
) -> dict: # mutable-ok: BaseConfig signature
mapped: Final = super().map_openai_params(
non_default_params=non_default_params,
optional_params=optional_params,
model=model,
drop_params=drop_params,
replace_max_completion_tokens_with_max_tokens=replace_max_completion_tokens_with_max_tokens,
)
malformed_effort: Final = non_string_reasoning_effort(non_default_params)
refused_while_reasoning: Final = chat_completions_params_refused_while_reasoning(model, non_default_params)
if malformed_effort and not (litellm.drop_params or drop_params):
raise litellm.utils.UnsupportedParamsError(
message=(
f"{model} takes reasoning_effort as a string on Bedrock's Chat Completions endpoint, not "
f"{type(non_default_params['reasoning_effort']).__name__}. Send one of its named efforts, or "
"set `litellm.drop_params = True` to drop it"
),
status_code=400,
)
if refused_while_reasoning and not (litellm.drop_params or drop_params):
raise litellm.utils.UnsupportedParamsError(
message=(
f"{model} doesn't support {sorted(refused_while_reasoning)} while reasoning is active on "
"Bedrock's Chat Completions endpoint. Set reasoning_effort to 'none' to send them, or set "
"`litellm.drop_params = True` to drop them"
),
status_code=400,
)
return dict( # mutable-ok: get_optional_params keeps filling this dict
without_refused_reasoning_effort(
model,
with_max_completion_tokens(_without_params(mapped, refused_while_reasoning | malformed_effort)),
)
)
def _inference_params(
self, optional_params: Mapping[str, object]
) -> dict[str, object]: # mutable-ok: BaseConfig signature of transform_request
return { # mutable-ok: OpenAILikeChatConfig.transform_request takes a plain dict
key: value
for key, value in optional_params.items()
if key not in self._aws_signer.aws_authentication_params
}
def transform_request(
self,
model: str,
messages: list[AllMessageValues], # mutable-ok: BaseConfig signature
optional_params: dict, # mutable-ok: BaseConfig signature
litellm_params: dict, # mutable-ok: BaseConfig signature
headers: dict, # mutable-ok: BaseConfig signature
) -> dict: # mutable-ok: BaseConfig signature
return super().transform_request(
model=split_bedrock_region_path(model)[1],
messages=inline_remote_media(messages, should_inline=inline_remote_image_urls),
optional_params=self._inference_params(optional_params),
litellm_params=litellm_params,
headers=headers,
)
async def async_transform_request(
self,
model: str,
messages: list[AllMessageValues], # mutable-ok: BaseConfig signature
optional_params: dict, # mutable-ok: BaseConfig signature
litellm_params: dict, # mutable-ok: BaseConfig signature
headers: dict, # mutable-ok: BaseConfig signature
) -> dict: # mutable-ok: BaseConfig signature
return await super().async_transform_request(
model=split_bedrock_region_path(model)[1],
messages=await async_inline_remote_media(messages, should_inline=inline_remote_image_urls),
optional_params=self._inference_params(optional_params),
litellm_params=litellm_params,
headers=headers,
)
def transform_response(
self,
model: str,
raw_response: httpx.Response,
model_response: ModelResponse,
logging_obj: "LiteLLMLoggingObj",
request_data: dict, # mutable-ok: BaseConfig signature
messages: list[AllMessageValues], # mutable-ok: BaseConfig signature
optional_params: dict, # mutable-ok: BaseConfig signature
litellm_params: dict, # mutable-ok: BaseConfig signature
encoding: "tiktoken.Encoding | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:
response: Final = super().transform_response(
model=model,
raw_response=raw_response,
model_response=model_response,
logging_obj=logging_obj,
request_data=request_data,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
encoding=encoding,
api_key=api_key,
json_mode=json_mode,
)
set_provider_response_headers_in_hidden_params(response, raw_response.headers)
for choice in response.choices:
if not isinstance(choice, Choices) or not isinstance(choice.message.content, str):
continue
reasoning, content = split_reasoning_tag(choice.message.content)
if reasoning is not None:
choice.message.reasoning_content = (
f"{getattr(choice.message, 'reasoning_content', None) or ''}{reasoning}"
)
choice.message.content = content
return response
def get_supported_openai_params(self, model: str) -> list: # mutable-ok: BaseConfig signature
refused: Final = frozenset(("n", *chat_completions_params_refused_for(model)))
base_params: Final = tuple(
param for param in super().get_supported_openai_params(model) if param not in refused
)
reasoning_param: Final = (
("reasoning_effort",)
if "reasoning_effort" not in base_params
and litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider)
else ()
)
return [*base_params, *reasoning_param] # mutable-ok: BaseConfig signature returns a list
def get_model_response_iterator(
self,
streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse,
sync_stream: bool,
json_mode: bool | None = False,
) -> BedrockRuntimeChatCompletionsStreamingHandler:
return BedrockRuntimeChatCompletionsStreamingHandler(
streaming_response=streaming_response,
sync_stream=sync_stream,
json_mode=json_mode,
)

218
litellm/llms/bedrock/common_utils.py
View file

@ -10,6 +10,7 @@ import json
import os
import re
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict
if TYPE_CHECKING:
@ -28,6 +29,7 @@ from litellm.llms.base_llm.anthropic_messages.transformation import (
)
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.bedrock.request_metadata import bedrock_request_metadata_is_owned
from litellm.secret_managers.main import get_secret, get_secret_str
from litellm.types.llms.bedrock import AWS_AUTH_PARAM_KEYS, AwsAuthParams
@ -37,6 +39,21 @@ if TYPE_CHECKING:
_ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs"
_OPENAI_FAMILY_MODEL_RE: Final = re.compile(r"(^|[./])openai\.")
_OPENAI_GPT_VERSION_RE: Final = re.compile(r"(^|[./])openai\.gpt-(\d{1,3})(?!\d)(?:\.(\d{1,3})(?!\d))?")
_BEDROCK_RUNTIME_CHAT_COMPLETIONS_DEFAULT_SINCE: Final = (5, 6)
_BEDROCK_RUNTIME_CHAT_COMPLETIONS_ENDPOINT: Final = "/v1/chat/completions"
BedrockRoute = Literal[
"converse",
"invoke",
"claude_platform",
"converse_like",
"agent",
"agentcore",
"async_invoke",
"openai",
"mantle",
"chat_completions",
]
def error_response_text(response: httpx.Response) -> str:
@ -787,12 +804,183 @@ def is_bedrock_application_inference_profile_arn(model: str) -> bool:
def strip_bedrock_routing_prefix(model: str) -> str:
"""Strip LiteLLM routing prefixes from model name."""
for prefix in ["bedrock/", "converse/", "invoke/", "openai/", "mantle/", "nova-2/", "nova/"]:
for prefix in ["bedrock/", "chat_completions/", "converse/", "invoke/", "openai/", "mantle/", "nova-2/", "nova/"]:
if model.startswith(prefix):
model = model.split("/", 1)[1]
return model
BEDROCK_CHAT_COMPLETIONS_ROUTE_PREFIX: Final = "chat_completions/"
BEDROCK_CONVERSE_ROUTE_PREFIX: Final = "converse/"
def without_bedrock_route_prefix(model: str) -> str:
return model.replace(BEDROCK_CONVERSE_ROUTE_PREFIX, "").replace(BEDROCK_CHAT_COMPLETIONS_ROUTE_PREFIX, "")
def split_bedrock_region_path(model: str) -> tuple[str | None, str]:
"""Split a ``<region>/<model-id>`` routing path into the region and the id AWS receives.
``bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0`` -> ``("us-gov-west-1", "openai.gpt-oss-20b-1:0")``;
a model without a region path comes back as ``(None, <routing-prefix-stripped id>)``.
"""
stripped: Final = strip_bedrock_routing_prefix(model)
region, separator, model_id = stripped.partition("/")
if separator and region in _get_all_bedrock_regions():
return region, model_id
return None, stripped
def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]:
return tuple(
litellm.model_cost.get(key)
for key in (model, strip_bedrock_routing_prefix(model), split_bedrock_region_path(model)[1])
)
def _bedrock_price_map_flag(model: str, flag: str) -> bool:
return any(entry is not None and entry.get(flag) is True for entry in _bedrock_price_map_entries(model))
def _price_map_entry_lists_endpoint(entry: Mapping[str, object] | None, endpoint: str) -> bool:
endpoints: Final = None if entry is None else entry.get("supported_endpoints")
return isinstance(endpoints, (list, tuple)) and endpoint in endpoints
def _openai_gpt_version(model: str) -> tuple[int, int] | None:
match: Final = _OPENAI_GPT_VERSION_RE.search(model)
if match is None:
return None
return int(match.group(2)), int(match.group(3) or 0)
def bedrock_runtime_chat_completions_is_default(model: str) -> bool:
"""Whether a model with no route prefix goes to bedrock-runtime's native Chat Completions by default.
GPT 5.6 and newer (``openai.gpt-<major>[.<minor>]`` at or above 5.6, which gpt-oss never matches) whose
price-map row lists ``/v1/chat/completions`` in ``supported_endpoints``. Older GPT rows, gpt-oss and Grok
stay on Converse unless the ``chat_completions/`` prefix opts them in.
"""
version: Final = _openai_gpt_version(model)
if version is None or version < _BEDROCK_RUNTIME_CHAT_COMPLETIONS_DEFAULT_SINCE:
return False
return any(
_price_map_entry_lists_endpoint(entry, _BEDROCK_RUNTIME_CHAT_COMPLETIONS_ENDPOINT)
for entry in _bedrock_price_map_entries(model)
)
def bedrock_runtime_chat_completions_serves_tools_with_reasoning(model: str) -> bool:
"""Whether AWS's native Chat Completions serves this model's function tools with any ``reasoning_effort``.
Data-driven from the price-map ``supports_bedrock_runtime_chat_completions_tools_with_reasoning``
flag (gpt-oss, Grok). Without it AWS only takes tools with ``reasoning_effort="none"``
(the GPT-5.6 family), and Converse serves tools with any effort, so those requests fall back to it.
"""
return _bedrock_price_map_flag(model, "supports_bedrock_runtime_chat_completions_tools_with_reasoning")
def bedrock_runtime_chat_completions_enforces_response_format(model: str) -> bool:
"""Whether AWS's native Chat Completions enforces a ``response_format`` schema for this model.
Data-driven from the price-map ``supports_bedrock_runtime_chat_completions_response_format`` flag
(GPT-5.6, Grok). Without it AWS accepts the field and answers with unconstrained text (gpt-oss), so
Converse, which emulates the schema through a forced ``json_tool_call`` tool, serves those requests.
"""
return _bedrock_price_map_flag(model, "supports_bedrock_runtime_chat_completions_response_format")
def bedrock_model_is_openai_gpt(model: str) -> bool:
"""A GPT-5.x or GPT-6.x id, never GPT-OSS: the families whose sampling params AWS ties to reasoning being off."""
return _openai_gpt_version(model) is not None
BEDROCK_CONVERSE_ONLY_REQUEST_KEYS: Final = frozenset(
(
"guardrailConfig",
"performanceConfig",
"serviceTier",
"requestMetadata",
"outputConfig",
"thinking",
"additionalModelRequestFields",
"top_k",
"stop",
"model_id",
)
)
def _response_format_needs_converse(model: str, response_format: object) -> bool:
if response_format is None:
return False
if not isinstance(response_format, Mapping):
return not bedrock_runtime_chat_completions_enforces_response_format(model)
response_format_type: Final = response_format.get("type")
if response_format_type == "text":
return False
is_json_schema: Final = response_format_type == "json_schema" and "json_schema" in response_format
return not (is_json_schema and bedrock_runtime_chat_completions_enforces_response_format(model))
def bedrock_request_needs_converse(model: str, request_params: Mapping[str, object]) -> bool:
"""Whether a request on the native Chat Completions route must still be served by Converse.
The route is the default for GPT 5.6 and newer (``bedrock_runtime_chat_completions_is_default``) and the
``chat_completions/`` prefix's opt-in for the rest; this decides the fallback for both alike.
Converse-shaped body keys (``BEDROCK_CONVERSE_ONLY_REQUEST_KEYS``, the Anthropic-style ``thinking``
block and the ``additionalModelRequestFields`` / ``top_k`` extension params included, which only Converse
forwards as ``additionalModelRequestFields`` and ``inferenceConfig``) have no field on
AWS's native OpenAI surface, a ``model_id`` override (an application inference profile or provisioned
throughput ARN) is only encoded into Converse's request URL and so stays on Converse like the
``bedrock/arn:...`` model form, ``stop`` stays on Converse where it fails loudly instead of silently
stopping hidden reasoning, operator-owned request metadata is only written onto the Converse body,
function tools (``tools`` or legacy ``functions``) on a model without
``supports_bedrock_runtime_chat_completions_tools_with_reasoning`` are rejected there unless
``reasoning_effort`` is exactly ``"none"``, and a ``response_format`` goes native only as
``{"type": "json_schema", "json_schema": ...}`` (a pydantic model is converted to that) on a model with
``supports_bedrock_runtime_chat_completions_response_format``: a schema on any other model is only
honored by Converse, and every ``json_object`` form (``response_schema`` included) keeps Converse's
handling everywhere, since AWS's native surface rejects that type with a 400 unless the prompt
mentions json.
"""
if any(request_params.get(key) is not None for key in BEDROCK_CONVERSE_ONLY_REQUEST_KEYS):
return True
if bedrock_request_metadata_is_owned():
return True
if _response_format_needs_converse(model, request_params.get("response_format")):
return True
if not (request_params.get("tools") or request_params.get("functions")):
return False
return (
not bedrock_runtime_chat_completions_serves_tools_with_reasoning(model)
and request_params.get("reasoning_effort") != "none"
)
def _chat_completions_unless_converse_needed(
model: str, request_params: Mapping[str, object] | None
) -> Literal["converse", "chat_completions"]:
if request_params is not None and bedrock_request_needs_converse(model, request_params):
return "converse"
return "chat_completions"
def bedrock_route_for_request(
model: str, request_params: Mapping[str, object], additional_drop_params: Sequence[str] | None
) -> BedrockRoute:
"""The route for one request, decided from the caller's raw params before any provider mapping.
Param mapping and dispatch both call this with the same inputs, so a request that falls back to
Converse is mapped with the Converse config and sent to Converse, never one without the other.
"""
dropped: Final = frozenset(additional_drop_params or ())
return BedrockModelInfo.get_bedrock_route(
model, MappingProxyType({key: value for key, value in request_params.items() if key not in dropped})
)
def strip_bedrock_throughput_suffix(model: str) -> str:
"""Strip throughput tier suffixes and context window suffixes from Bedrock model names."""
import re
@ -1150,19 +1338,16 @@ class BedrockModelInfo(BaseLLMModelInfo):
@staticmethod
def get_bedrock_route(
model: str,
) -> Literal[
"converse",
"invoke",
"claude_platform",
"converse_like",
"agent",
"agentcore",
"async_invoke",
"openai",
"mantle",
]:
request_params: Mapping[str, object] | None = None,
) -> BedrockRoute:
"""
Get the bedrock route for the given model.
GPT 5.6 and newer go to bedrock-runtime's native OpenAI Chat Completions by default
(``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in;
``request_params`` (the caller's chat params) sends such a request to Converse when it needs a
feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family
model stays on Converse without the prefix.
"""
route_mappings: dict[
str,
@ -1176,6 +1361,7 @@ class BedrockModelInfo(BaseLLMModelInfo):
"async_invoke",
"openai",
"mantle",
"chat_completions",
],
] = {
"invoke/": "invoke",
@ -1197,6 +1383,9 @@ class BedrockModelInfo(BaseLLMModelInfo):
if BedrockModelInfo._model_has_route_prefix(model, prefix):
return route_type
if BedrockModelInfo._model_has_route_prefix(model, "chat_completions/"):
return _chat_completions_unless_converse_needed(model, request_params)
# Check for nova spec prefixes (nova/ and nova-2/)
_model_after_bedrock: Final = model.replace("bedrock/", "", 1)
if _model_after_bedrock.startswith("nova-2/") or _model_after_bedrock.startswith("nova/"):
@ -1205,6 +1394,9 @@ class BedrockModelInfo(BaseLLMModelInfo):
if is_bedrock_application_inference_profile_arn(model):
return "converse"
if bedrock_runtime_chat_completions_is_default(model):
return _chat_completions_unless_converse_needed(model, request_params)
base_model: Final = BedrockModelInfo.get_base_model(model)
alt_model: Final = BedrockModelInfo.get_non_litellm_routing_model_name(model=model)
if base_model in litellm.bedrock_converse_models or alt_model in litellm.bedrock_converse_models:
@ -1383,6 +1575,8 @@ def get_bedrock_chat_config(model: str):
return litellm.AmazonConverseConfig()
elif bedrock_route == "openai":
return litellm.AmazonBedrockOpenAIConfig()
elif bedrock_route == "chat_completions":
return litellm.AmazonBedrockRuntimeChatCompletionsConfig()
elif bedrock_route == "agent":
from litellm.llms.bedrock.chat.invoke_agent.transformation import (
AmazonInvokeAgentConfig,

15
litellm/llms/bedrock/responses/transformation.py
View file

@ -50,6 +50,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import (
BEDROCK_CHAT_COMPLETIONS_ROUTE_PREFIX,
BedrockError,
bedrock_supports_openai_responses,
)
@ -76,6 +77,10 @@ IMAGE_BLOCK_KEYS: Final = ("content", "output")
IMAGE_BLOCK_TYPES: Final = frozenset({"input_image", "computer_screenshot"})
def _without_chat_completions_route(model: str) -> str:
return model.removeprefix(BEDROCK_CHAT_COMPLETIONS_ROUTE_PREFIX)
def resolve_bedrock_bearer_token(api_key: str | None) -> str | None:
return api_key or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
@ -168,9 +173,13 @@ class BedrockOpenAIResponsesConfig(BaseAWSLLM, OpenAIResponsesAPIConfig):
The capability decision lives here rather than in the shared dispatch so that
onboarding a model, or changing how the signal is read, stays inside the
Bedrock adapter. ``None`` leaves the caller's existing behaviour untouched --
chat-only Bedrock models keep the Chat Completions bridge.
chat-only Bedrock models keep the Chat Completions bridge. The ``chat_completions/``
opt-in only moves Chat Completions calls off Converse, so a Responses call on such a
deployment still takes this surface instead of being bridged.
"""
if not bedrock_supports_openai_responses(model, litellm.model_cost):
if not model or not bedrock_supports_openai_responses(
_without_chat_completions_route(model), litellm.model_cost
):
return None
return cls()
@ -328,7 +337,7 @@ class BedrockOpenAIResponsesConfig(BaseAWSLLM, OpenAIResponsesAPIConfig):
rewritten_types,
)
return super().transform_responses_api_request(
model=model,
model=_without_chat_completions_route(model),
input=normalized_input,
response_api_optional_request_params=response_api_optional_request_params,
litellm_params=litellm_params,

13
litellm/main.py
View file

@ -116,7 +116,11 @@ from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig
from litellm.llms.base_llm.base_model_iterator import (
convert_model_response_to_streaming,
)
from litellm.llms.bedrock.common_utils import BedrockModelInfo
from litellm.llms.bedrock.common_utils import (
BedrockModelInfo,
bedrock_route_for_request,
without_bedrock_route_prefix,
)
from litellm.llms.cohere.common_utils import CohereModelInfo
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
@ -4205,7 +4209,9 @@ def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes
if "aws_region_name" not in optional_params or optional_params["aws_region_name"] is None:
optional_params["aws_region_name"] = aws_bedrock_client.meta.region_name
bedrock_route: Final = BedrockModelInfo.get_bedrock_route(model)
bedrock_route: Final = bedrock_route_for_request(
model, ctx.request_params, ctx.kwargs.get("additional_drop_params")
)
if bedrock_route == "claude_platform":
provider_config = ProviderConfigManager.get_provider_chat_config(
model=model,
@ -4232,7 +4238,7 @@ def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes
provider_config=provider_config,
)
elif bedrock_route == "converse":
model = model.replace("converse/", "")
model = without_bedrock_route_prefix(model)
response = bedrock_converse_chat_completion.completion(
model=model,
messages=messages,
@ -5841,6 +5847,7 @@ def completion(
optional_params=optional_params,
organization=organization,
provider_config=provider_config,
request_params=MappingProxyType({**optional_param_args, **non_default_params}),
shared_session=shared_session,
stream=stream,
temperature=temperature,

86
litellm/model_prices_and_context_window_backup.json
View file

@ -41652,6 +41652,10 @@
"output_cost_per_token": 0.0
},
"openai.gpt-oss-120b-1:0": {
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 128000,
@ -41666,6 +41670,10 @@
"supports_tool_choice": true
},
"openai.gpt-oss-20b-1:0": {
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"input_cost_per_token": 7e-08,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 128000,
@ -47406,6 +47414,10 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.6e-07,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -47419,6 +47431,10 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -47428,6 +47444,11 @@
"input_cost_per_token": 2.64e-06,
"output_cost_per_token": 7.92e-06,
"cache_read_input_token_cost": 6.6e-07,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 500000,
"max_output_tokens": 500000,
@ -58032,6 +58053,7 @@
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html"
},
"us.openai.gpt-5.6-sol": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 4.4e-06,
"input_cost_per_token_above_272k_tokens": 8.8e-06,
"cache_creation_input_token_cost": 5.5e-06,
@ -58062,10 +58084,12 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"global.openai.gpt-5.6-sol": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 4e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
"cache_creation_input_token_cost": 5e-06,
@ -58096,10 +58120,12 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"us.openai.gpt-5.6-terra": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2.2e-06,
"input_cost_per_token_above_272k_tokens": 4.4e-06,
"cache_creation_input_token_cost": 2.75e-06,
@ -58130,10 +58156,12 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"global.openai.gpt-5.6-terra": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
"cache_creation_input_token_cost": 2.5e-06,
@ -58164,10 +58192,12 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"us.openai.gpt-5.6-luna": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2.2e-07,
"input_cost_per_token_above_272k_tokens": 4.4e-07,
"cache_creation_input_token_cost": 2.75e-07,
@ -58198,6 +58228,7 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
@ -58326,6 +58357,7 @@
]
},
"global.openai.gpt-5.6-luna": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2e-07,
"input_cost_per_token_above_272k_tokens": 4e-07,
"cache_creation_input_token_cost": 2.5e-07,
@ -58356,6 +58388,7 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
@ -58474,6 +58507,7 @@
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html"
},
"us.openai.gpt-6-astra": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 1.1e-05,
"input_cost_per_token_above_272k_tokens": 2.2e-05,
"cache_creation_input_token_cost": 1.375e-05,
@ -58503,12 +58537,15 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"us.openai.gpt-6-sol": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2.2e-06,
"input_cost_per_token_above_272k_tokens": 4.4e-06,
"cache_creation_input_token_cost": 2.75e-06,
@ -58538,12 +58575,15 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"us.openai.gpt-6-luna": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 1.1e-07,
"input_cost_per_token_above_272k_tokens": 2.2e-07,
"cache_creation_input_token_cost": 1.375e-07,
@ -58573,12 +58613,15 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"global.openai.gpt-6-astra": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 1e-05,
"input_cost_per_token_above_272k_tokens": 2e-05,
"cache_creation_input_token_cost": 1.25e-05,
@ -58608,8 +58651,10 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
@ -58643,9 +58688,11 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"global.openai.gpt-6-sol": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
"cache_creation_input_token_cost": 2.5e-06,
@ -58675,8 +58722,10 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
@ -58710,9 +58759,11 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"global.openai.gpt-6-luna": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 1e-07,
"input_cost_per_token_above_272k_tokens": 2e-07,
"cache_creation_input_token_cost": 1.25e-07,
@ -58742,8 +58793,10 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
@ -59040,6 +59093,11 @@
"input_cost_per_token": 2.2e-06,
"output_cost_per_token": 6.6e-06,
"cache_read_input_token_cost": 5.5e-07,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 500000,
"max_output_tokens": 500000,
@ -59056,6 +59114,11 @@
"input_cost_per_token": 2e-06,
"output_cost_per_token": 6e-06,
"cache_read_input_token_cost": 5e-07,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 500000,
"max_output_tokens": 500000,
@ -65042,6 +65105,10 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.6e-07,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -65055,6 +65122,10 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -65296,6 +65367,10 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.6e-07,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -65309,6 +65384,10 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -79432,6 +79511,7 @@
"output_cost_per_token_above_272k_tokens": 1.5e-05,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supported_modalities": [
@ -79441,6 +79521,7 @@
"supported_output_modalities": [
"text"
],
"supports_bedrock_runtime_chat_completions_response_format": true,
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
@ -79449,6 +79530,7 @@
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_sampling_params": false,
"supports_xhigh_reasoning_effort": true
},
"openai.gpt-6.1-sol": {
@ -79481,6 +79563,7 @@
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_sampling_params": false,
"supports_xhigh_reasoning_effort": true
},
"bedrock_mantle/openai.gpt-6.1-sol": {
@ -79537,6 +79620,7 @@
"output_cost_per_token_above_272k_tokens": 1.65e-05,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supported_modalities": [
@ -79546,6 +79630,7 @@
"supported_output_modalities": [
"text"
],
"supports_bedrock_runtime_chat_completions_response_format": true,
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
@ -79554,6 +79639,7 @@
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_sampling_params": false,
"supports_xhigh_reasoning_effort": true
},
"vertex_ai/gemini-3.8-flash-tts": {

3
litellm/types/completion.py
View file

@ -1,6 +1,6 @@
from __future__ import annotations
from collections.abc import Callable, Coroutine, Iterable
from collections.abc import Callable, Coroutine, Iterable, Mapping
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Literal, Union
@ -229,6 +229,7 @@ class _CompletionDispatchContext:
optional_params: dict
organization: str | None
provider_config: BaseConfig | None
request_params: Mapping[str, object]
shared_session: ClientSession | None
stream: bool | None
temperature: float | None

23
litellm/utils.py
View file

@ -410,7 +410,7 @@ if TYPE_CHECKING:
BaseVectorStoreFilesConfig,
)
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
from litellm.llms.bedrock.common_utils import BedrockModelInfo
from litellm.llms.bedrock.common_utils import BedrockModelInfo, BedrockRoute
from litellm.llms.bedrock.embed.amazon_nova_transformation import (
AmazonNovaEmbeddingConfig,
)
@ -3473,6 +3473,14 @@ def _should_drop_param(k, additional_drop_params) -> bool:
return False
def _bedrock_route_for_request(
model: str, passed_params: Mapping[str, object], additional_drop_params: list | None
) -> BedrockRoute:
from litellm.llms.bedrock.common_utils import bedrock_route_for_request
return bedrock_route_for_request(model, passed_params, additional_drop_params)
def _get_non_default_params(passed_params: dict, default_params: dict, additional_drop_params: list | None) -> dict:
non_default_params: Final = {}
for k, v in passed_params.items():
@ -4514,9 +4522,17 @@ def get_optional_params(
message=f"{custom_llm_provider} does not support parameters: {list(unsupported_params.keys())}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n. \n If you want to use these params dynamically send allowed_openai_params={list(unsupported_params.keys())} in your request.",
)
bedrock_route: Final = (
_bedrock_route_for_request(model, passed_params, additional_drop_params)
if custom_llm_provider == "bedrock"
else None
)
get_supported_openai_params: Final[_SupportedOpenAIParamsGetter] = litellm_utils.get_supported_openai_params
supported_params = get_supported_openai_params(
model=model, custom_llm_provider=custom_llm_provider, base_model=base_model
supported_params = (
litellm.AmazonConverseConfig().get_supported_openai_params(model=model)
if bedrock_route == "converse"
and isinstance(provider_config, litellm.AmazonBedrockRuntimeChatCompletionsConfig)
else get_supported_openai_params(model=model, custom_llm_provider=custom_llm_provider, base_model=base_model)
)
if supported_params is None:
supported_params = get_supported_openai_params(model=model, custom_llm_provider="openai")
@ -4686,7 +4702,6 @@ def get_optional_params(
)
elif custom_llm_provider == "bedrock":
bedrock_model_info: Final[type[BedrockModelInfo]] = litellm_utils.BedrockModelInfo
bedrock_route: Final = bedrock_model_info.get_bedrock_route(model)
bedrock_base_model: Final = bedrock_model_info.get_base_model(model)
if bedrock_route == "converse" or bedrock_route == "converse_like":
optional_params = litellm.AmazonConverseConfig().map_openai_params(

86
model_prices_and_context_window.json
View file

@ -41652,6 +41652,10 @@
"output_cost_per_token": 0.0
},
"openai.gpt-oss-120b-1:0": {
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 128000,
@ -41666,6 +41670,10 @@
"supports_tool_choice": true
},
"openai.gpt-oss-20b-1:0": {
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"input_cost_per_token": 7e-08,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 128000,
@ -47406,6 +47414,10 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.6e-07,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -47419,6 +47431,10 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -47428,6 +47444,11 @@
"input_cost_per_token": 2.64e-06,
"output_cost_per_token": 7.92e-06,
"cache_read_input_token_cost": 6.6e-07,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 500000,
"max_output_tokens": 500000,
@ -58032,6 +58053,7 @@
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html"
},
"us.openai.gpt-5.6-sol": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 4.4e-06,
"input_cost_per_token_above_272k_tokens": 8.8e-06,
"cache_creation_input_token_cost": 5.5e-06,
@ -58062,10 +58084,12 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"global.openai.gpt-5.6-sol": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 4e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
"cache_creation_input_token_cost": 5e-06,
@ -58096,10 +58120,12 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"us.openai.gpt-5.6-terra": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2.2e-06,
"input_cost_per_token_above_272k_tokens": 4.4e-06,
"cache_creation_input_token_cost": 2.75e-06,
@ -58130,10 +58156,12 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"global.openai.gpt-5.6-terra": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
"cache_creation_input_token_cost": 2.5e-06,
@ -58164,10 +58192,12 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"us.openai.gpt-5.6-luna": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2.2e-07,
"input_cost_per_token_above_272k_tokens": 4.4e-07,
"cache_creation_input_token_cost": 2.75e-07,
@ -58198,6 +58228,7 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
@ -58326,6 +58357,7 @@
]
},
"global.openai.gpt-5.6-luna": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2e-07,
"input_cost_per_token_above_272k_tokens": 4e-07,
"cache_creation_input_token_cost": 2.5e-07,
@ -58356,6 +58388,7 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
@ -58474,6 +58507,7 @@
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html"
},
"us.openai.gpt-6-astra": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 1.1e-05,
"input_cost_per_token_above_272k_tokens": 2.2e-05,
"cache_creation_input_token_cost": 1.375e-05,
@ -58503,12 +58537,15 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"us.openai.gpt-6-sol": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2.2e-06,
"input_cost_per_token_above_272k_tokens": 4.4e-06,
"cache_creation_input_token_cost": 2.75e-06,
@ -58538,12 +58575,15 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"us.openai.gpt-6-luna": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 1.1e-07,
"input_cost_per_token_above_272k_tokens": 2.2e-07,
"cache_creation_input_token_cost": 1.375e-07,
@ -58573,12 +58613,15 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"global.openai.gpt-6-astra": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 1e-05,
"input_cost_per_token_above_272k_tokens": 2e-05,
"cache_creation_input_token_cost": 1.25e-05,
@ -58608,8 +58651,10 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
@ -58643,9 +58688,11 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"global.openai.gpt-6-sol": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
"cache_creation_input_token_cost": 2.5e-06,
@ -58675,8 +58722,10 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
@ -58710,9 +58759,11 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"global.openai.gpt-6-luna": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 1e-07,
"input_cost_per_token_above_272k_tokens": 2e-07,
"cache_creation_input_token_cost": 1.25e-07,
@ -58742,8 +58793,10 @@
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true,
"supports_sampling_params": false,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
@ -59040,6 +59093,11 @@
"input_cost_per_token": 2.2e-06,
"output_cost_per_token": 6.6e-06,
"cache_read_input_token_cost": 5.5e-07,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 500000,
"max_output_tokens": 500000,
@ -59056,6 +59114,11 @@
"input_cost_per_token": 2e-06,
"output_cost_per_token": 6e-06,
"cache_read_input_token_cost": 5e-07,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 500000,
"max_output_tokens": 500000,
@ -65042,6 +65105,10 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.6e-07,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -65055,6 +65122,10 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -65296,6 +65367,10 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.6e-07,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -65309,6 +65384,10 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -79432,6 +79511,7 @@
"output_cost_per_token_above_272k_tokens": 1.5e-05,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supported_modalities": [
@ -79441,6 +79521,7 @@
"supported_output_modalities": [
"text"
],
"supports_bedrock_runtime_chat_completions_response_format": true,
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
@ -79449,6 +79530,7 @@
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_sampling_params": false,
"supports_xhigh_reasoning_effort": true
},
"openai.gpt-6.1-sol": {
@ -79481,6 +79563,7 @@
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_sampling_params": false,
"supports_xhigh_reasoning_effort": true
},
"bedrock_mantle/openai.gpt-6.1-sol": {
@ -79537,6 +79620,7 @@
"output_cost_per_token_above_272k_tokens": 1.65e-05,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supported_modalities": [
@ -79546,6 +79630,7 @@
"supported_output_modalities": [
"text"
],
"supports_bedrock_runtime_chat_completions_response_format": true,
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
@ -79554,6 +79639,7 @@
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_sampling_params": false,
"supports_xhigh_reasoning_effort": true
},
"vertex_ai/gemini-3.8-flash-tts": {

6
model_prices_and_context_window.schema.json
View file

@ -990,6 +990,12 @@
"supports_audio_output": {
"type": "boolean"
},
"supports_bedrock_runtime_chat_completions_response_format": {
"type": "boolean"
},
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": {
"type": "boolean"
},
"supports_computer_use": {
"type": "boolean"
},

276
tests/integration/_support/bedrock_runtime_peer.py Normal file
View file

@ -0,0 +1,276 @@
import json
import re
import threading
from collections.abc import Mapping
from multiprocessing.sharedctypes import Synchronized
from types import MappingProxyType
from typing import Final
from urllib.parse import unquote
from integration._support.upstream import _aws_event_frame
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
MARKER: Final = re.compile(r"marker-([0-9a-f]{32})")
EVENT_STREAM: Final = "application/vnd.amazon.eventstream"
REASONING_EFFORTS: Final = frozenset(("none", "minimal", "low", "medium", "high", "xhigh"))
NATIVE_CHAT: Final = "/openai/v1/chat/completions"
NATIVE_RESPONSES: Final = "/openai/v1/responses"
PNG_1X1: Final = bytes.fromhex(
"89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c489"
"0000000d49444154789c63f8cfc0f01f00050001ff89993d1d0000000049454e44ae426082"
)
USAGE: Final[Mapping[str, JsonValue]] = MappingProxyType(
{
"prompt_tokens": 9,
"completion_tokens": 5,
"total_tokens": 14,
"completion_tokens_details": {"reasoning_tokens": 3},
}
)
_STATUS: Final = re.compile(r"status=(\d{3})")
_CONVERSE: Final = re.compile(r"^/model/(.+)/converse$")
_CONVERSE_STREAM: Final = re.compile(r"^/model/(.+)/converse-stream$")
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_NO_MARKER: Final = "0" * 32
def marker_of(request: Request) -> str:
found: Final = MARKER.search(request.body.decode(errors="replace"))
return _NO_MARKER if found is None else found.group(1)
def body_of(request: Request) -> Mapping[str, JsonValue]:
try:
return _JSON_OBJECT.validate_json(request.body)
except ValueError:
return {}
def target_of(request: Request) -> str:
return unquote(request.target)
def answer(marker: str) -> str:
return f"answer marker-{marker}"
def reasoning_answer(marker: str) -> str:
return f"<reasoning>why marker-{marker}</reasoning> {answer(marker)}"
def _headers(marker: str) -> Mapping[str, str]:
return MappingProxyType({"x-amzn-requestid": marker})
def _json_reply(status: int, payload: Mapping[str, JsonValue], marker: str) -> Reply:
return Reply(status=status, body=json.dumps(payload).encode(), headers=_headers(marker))
def _error(status: int, message: str, marker: str) -> Reply:
return _json_reply(status, {"message": message}, marker)
def _effort_of(target: str, body: Mapping[str, JsonValue]) -> JsonValue:
if not _CONVERSE.match(target) and not _CONVERSE_STREAM.match(target):
return body.get("reasoning_effort")
fields: Final = body.get("additionalModelRequestFields")
reasoning: Final = fields.get("reasoning") if isinstance(fields, Mapping) else None
return reasoning.get("effort") if isinstance(reasoning, Mapping) else None
def forwarded_effort(request: Request) -> JsonValue:
return _effort_of(target_of(request), body_of(request))
def _sse(frames: tuple[Mapping[str, JsonValue], ...], pause: float) -> Reply:
return Reply(
content_type="text/event-stream",
chunks=(*(b"data: " + json.dumps(frame).encode() + b"\n\n" for frame in frames), b"data: [DONE]\n\n"),
pause_between_chunks=pause,
)
def _with_headers(reply: Reply, marker: str) -> Reply:
return Reply(
status=reply.status,
body=reply.body,
content_type=reply.content_type,
chunks=reply.chunks,
abort_after=reply.abort_after,
gate_after_first=reply.gate_after_first,
pause_between_chunks=reply.pause_between_chunks,
headers=_headers(marker),
)
def _content_deltas(model: str, marker: str) -> tuple[str, ...]:
if "gpt-oss" in model:
return ("<reason", "ing>why ", f"marker-{marker}", "</reas", "oning> answer ", f"marker-{marker}")
return ("answer ", f"marker-{marker}")
def _chat_text(model: str, marker: str) -> str:
return reasoning_answer(marker) if "gpt-oss" in model else answer(marker)
def _chat_reply(model: str, marker: str, stream: bool, pause: float) -> Reply:
identity: Final = f"chatcmpl-{marker}"
if not stream:
return _json_reply(
200,
{
"id": identity,
"object": "chat.completion",
"created": 1,
"model": model,
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": _chat_text(model, marker)},
"finish_reason": "stop",
}
],
"usage": dict(USAGE),
},
marker,
)
deltas: Final = _content_deltas(model, marker)
frames: Final = tuple(
{
"id": identity,
"object": "chat.completion.chunk",
"created": 1,
"model": model,
"choices": [{"index": 0, "delta": {"role": "assistant", "content": delta}, "finish_reason": None}],
}
for delta in deltas
)
finish: Final[Mapping[str, JsonValue]] = {
"id": identity,
"object": "chat.completion.chunk",
"created": 1,
"model": model,
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
"usage": dict(USAGE),
}
return _with_headers(_sse((*frames, finish), pause), marker)
def _responses_reply(model: str, marker: str, stream: bool, pause: float) -> Reply:
identity: Final = f"resp_upstream_{marker}"
item_id: Final = f"msg_{marker}"
response: Final[Mapping[str, JsonValue]] = {
"id": identity,
"object": "response",
"created_at": 1,
"status": "completed",
"model": model,
"output": [
{
"type": "message",
"id": item_id,
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": answer(marker), "annotations": []}],
}
],
"usage": {"input_tokens": 30, "output_tokens": 5, "total_tokens": 35},
}
if not stream:
return _json_reply(200, response, marker)
events: Final[tuple[Mapping[str, JsonValue], ...]] = (
{
"type": "response.created",
"sequence_number": 0,
"response": {**response, "status": "in_progress", "output": []},
},
{
"type": "response.output_text.delta",
"sequence_number": 1,
"item_id": item_id,
"output_index": 0,
"content_index": 0,
"delta": answer(marker),
},
{"type": "response.completed", "sequence_number": 2, "response": response},
)
return Reply(
content_type="text/event-stream",
chunks=tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events),
pause_between_chunks=pause,
headers=_headers(marker),
)
def _converse_reply(marker: str) -> Reply:
return _json_reply(
200,
{
"output": {"message": {"role": "assistant", "content": [{"text": answer(marker)}]}},
"stopReason": "end_turn",
"usage": {"inputTokens": 9, "outputTokens": 5, "totalTokens": 14},
"metrics": {"latencyMs": 1},
},
marker,
)
def _converse_stream_reply(marker: str, pause: float) -> Reply:
events: Final[tuple[tuple[str, Mapping[str, JsonValue]], ...]] = (
("messageStart", {"role": "assistant"}),
("contentBlockDelta", {"delta": {"text": "answer "}, "contentBlockIndex": 0}),
("contentBlockDelta", {"delta": {"text": f"marker-{marker}"}, "contentBlockIndex": 0}),
("contentBlockStop", {"contentBlockIndex": 0}),
("messageStop", {"stopReason": "end_turn"}),
("metadata", {"usage": {"inputTokens": 9, "outputTokens": 5, "totalTokens": 14}, "metrics": {"latencyMs": 1}}),
)
return Reply(
content_type=EVENT_STREAM,
chunks=tuple(_aws_event_frame(kind, payload, "sc", marker) for kind, payload in events),
pause_between_chunks=pause,
headers=_headers(marker),
)
def respond(request: Request, *, pause: float = 0.0) -> Reply:
target: Final = target_of(request)
marker: Final = marker_of(request)
if request.method == "GET":
if target == "/image.png":
return Reply(body=PNG_1X1, content_type="image/png", headers=_headers(marker))
return _error(404, f"no scripted object at {target}", marker)
scripted_status: Final = _STATUS.search(request.body.decode(errors="replace"))
if scripted_status is not None:
status: Final = int(scripted_status.group(1))
return _error(status, f"scripted {status}", marker)
body: Final = body_of(request)
effort: Final = _effort_of(target, body)
if effort is not None and (not isinstance(effort, str) or effort not in REASONING_EFFORTS):
return _error(400, f"Invalid reasoning effort: {json.dumps(effort)}", marker)
model: Final = str(body.get("model", ""))
stream: Final = body.get("stream") is True
if request.method == "POST" and target == NATIVE_CHAT:
return _chat_reply(model, marker, stream, pause)
if request.method == "POST" and target == NATIVE_RESPONSES:
return _responses_reply(model, marker, stream, pause)
if request.method == "POST" and _CONVERSE.match(target):
return _converse_reply(marker)
if request.method == "POST" and _CONVERSE_STREAM.match(target):
return _converse_stream_reply(marker, pause)
return _error(404, f"unknown bedrock route {request.method} {target}", marker)
def serve_peer(port: int, received: Synchronized[int], answer_first: int) -> None:
held: Final = threading.Event()
def respond_or_hold(request: Request) -> Reply:
with received.get_lock():
received.value += 1
ordinal: Final = received.value
if ordinal > answer_first:
held.wait()
return respond(request)
with wire_server(respond_or_hold, port=port):
threading.Event().wait()

194
tests/integration/messages_endpoint/providers/bedrock/test_bedrock_messages_gpt_chat_completions_wire.py Normal file
View file

@ -0,0 +1,194 @@
import json
import uuid
from collections.abc import Mapping
from typing import Final
import anthropic
from integration._support.bedrock_runtime_peer import NATIVE_CHAT, answer, body_of, marker_of, respond, target_of
from integration._support.client import Gateway, Scenario, eventually
from integration._support.database import read_rows
from integration._support.wire import Request, Wire, wire_server
from pydantic import JsonValue
BEDROCK_MODEL: Final = "us.openai.gpt-5.6-sol"
TOKEN: Final = "synthetic-bedrock-bearer"
NO_CACHE: Final[Mapping[str, JsonValue]] = {"cache": {"no-cache": True}}
ANTHROPIC_VERSION: Final[Mapping[str, str]] = {"anthropic-version": "2023-06-01"}
def _question(marker: str) -> str:
return f"Question marker-{marker}"
def _deployment(scenario: Scenario, wire: Wire) -> str:
return scenario.model(
model=f"bedrock/{BEDROCK_MODEL}",
api_key=TOKEN,
aws_region_name="us-east-1",
aws_bedrock_runtime_endpoint=wire.url,
)
def _carrying(wire: Wire, marker: str) -> tuple[Request, ...]:
return tuple(request for request in wire.drain() if marker_of(request) == marker)
def _native_body(wire: Wire, marker: str) -> Mapping[str, JsonValue]:
received: Final = _carrying(wire, marker)
assert [(request.method, target_of(request)) for request in received] == [("POST", NATIVE_CHAT)]
assert received[0].headers["authorization"] == f"Bearer {TOKEN}", received[0].headers
return body_of(received[0])
def _native_request(marker: str, max_tokens: int, effort: str) -> Mapping[str, JsonValue]:
return {
"model": BEDROCK_MODEL,
"messages": [{"role": "user", "content": _question(marker)}],
"max_completion_tokens": max_tokens,
"reasoning_effort": effort,
}
def _spend_rows(identity: str, expected: int) -> list[dict[str, JsonValue]]:
return eventually(
lambda: read_rows(
"SELECT request_id, call_type, status, model_group, prompt_tokens, completion_tokens, cache_hit"
' FROM "LiteLLM_SpendLogs" WHERE starts_with(request_id, %s) ORDER BY "startTime"',
(identity,),
),
lambda found: len(found) == expected,
seconds=70,
)
def _success_row(identity: str, model: str, cache_hit: str = "None") -> dict[str, JsonValue]:
return {
"request_id": identity,
"call_type": "anthropic_messages",
"status": "success",
"model_group": model,
"prompt_tokens": 9,
"completion_tokens": 5,
"cache_hit": cache_hit,
}
def test_anthropic_sdk_thinking_budget_reaches_native_chat_completions_as_reasoning_effort(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
client: Final = anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0)
message: Final = client.messages.create(
model=model,
max_tokens=4096,
thinking={"type": "enabled", "budget_tokens": 2048},
messages=[{"role": "user", "content": _question(marker)}],
extra_body=NO_CACHE,
)
assert _native_body(wire, marker) == _native_request(marker, 4096, "medium")
assert message.id == f"chatcmpl-{marker}", message
assert [(block.type, getattr(block, "text", None)) for block in message.content] == [("text", answer(marker))]
assert (message.usage.input_tokens, message.usage.output_tokens) == (9, 5), message
assert _spend_rows(message.id, 1) == [_success_row(message.id, model)]
def test_anthropic_sdk_stream_with_thinking_budget_is_served_by_native_chat_completions(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
client: Final = anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0)
stream: Final = client.messages.create(
model=model,
max_tokens=4096,
thinking={"type": "enabled", "budget_tokens": 2048},
messages=[{"role": "user", "content": _question(marker)}],
extra_body=NO_CACHE,
stream=True,
)
events: Final = list(stream)
assert _native_body(wire, marker) == {
**_native_request(marker, 4096, "medium"),
"stream": True,
"stream_options": {"include_usage": True},
}
assert events[0].type == "message_start" and events[-1].type == "message_stop", events
identity: Final = events[0].message.id
assert identity.startswith("msg_"), events
assert "".join(
event.delta.text
for event in events
if event.type == "content_block_delta" and event.delta.type == "text_delta"
) == answer(marker)
assert _spend_rows(identity, 1) == [_success_row(identity, model, cache_hit="False")]
def test_raw_thinking_summary_reaches_native_chat_completions_as_the_plain_effort(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = gateway.request(
"POST",
"/v1/messages",
{
"model": model,
"max_tokens": 4096,
"thinking": {"type": "enabled", "budget_tokens": 2048, "summary": "detailed"},
"messages": [{"role": "user", "content": _question(marker)}],
**NO_CACHE,
},
headers=ANTHROPIC_VERSION,
)
body: Final = _native_body(wire, marker)
assert body == _native_request(marker, 4096, "medium")
assert "summary" not in json.dumps(body), body
assert response.status_code == 200, response.text
assert response.json()["id"] == f"chatcmpl-{marker}", response.text
assert response.json()["content"] == [{"type": "text", "text": answer(marker)}], response.text
assert _spend_rows(f"chatcmpl-{marker}", 1) == [_success_row(f"chatcmpl-{marker}", model)]
async def test_async_anthropic_sdk_disabled_thinking_reaches_native_chat_completions_as_effort_none(
gateway: Gateway,
) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
client: Final = anthropic.AsyncAnthropic(
base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0
)
message: Final = await client.messages.create(
model=model,
max_tokens=64,
thinking={"type": "disabled"},
messages=[{"role": "user", "content": _question(marker)}],
extra_body=NO_CACHE,
)
assert _native_body(wire, marker) == _native_request(marker, 64, "none")
assert message.id == f"chatcmpl-{marker}", message
assert [(block.type, getattr(block, "text", None)) for block in message.content] == [("text", answer(marker))]
assert _spend_rows(message.id, 1) == [_success_row(message.id, model)]
def test_identical_messages_requests_reach_the_peer_once_and_log_a_cache_hit_row(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
body: Final[dict[str, JsonValue]] = {
"model": model,
"max_tokens": 64,
"messages": [{"role": "user", "content": _question(marker)}],
}
first: Final = gateway.request("POST", "/v1/messages", body, headers=ANTHROPIC_VERSION)
assert first.status_code == 200, first.text
identity: Final = str(first.json()["id"])
assert first.json()["content"] == [{"type": "text", "text": answer(marker)}], first.text
second: Final = gateway.request("POST", "/v1/messages", body, headers=ANTHROPIC_VERSION)
assert second.status_code == 200, second.text
assert second.json()["id"] == identity, (first.text, second.text)
assert second.json()["content"] == [{"type": "text", "text": answer(marker)}], second.text
received: Final = _carrying(wire, marker)
assert [(request.method, marker_of(request)) for request in received] == [("POST", marker)], received
rows: Final = _spend_rows(identity, 2)
assert rows[0] == _success_row(identity, model), rows
assert str(rows[1]["request_id"]).startswith(identity + "_cache_hit"), rows
assert {**rows[1], "request_id": identity, "cache_hit": "None"} == _success_row(identity, model), rows

123
tests/integration/providers/test_bedrock_gpt_responses_native_wire.py Normal file
View file

@ -0,0 +1,123 @@
import base64
import uuid
from dataclasses import dataclass
from typing import Final
import openai
from integration._support.bedrock_runtime_peer import NATIVE_RESPONSES, answer, respond, target_of
from integration._support.client import Gateway, Scenario, eventually
from integration._support.database import read_rows
from integration._support.wire import Request, Wire, wire_server
from openai.types.responses import ResponseCompletedEvent, ResponseTextDeltaEvent
from pydantic import JsonValue, TypeAdapter
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_if_encrypted_with
GPT: Final = "us.openai.gpt-5.6-sol"
TOKEN: Final = "synthetic-bedrock-bearer"
SALT: Final = "sk-integration-salt"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
@dataclass(frozen=True, slots=True)
class _IssuedId:
issued: str
upstream: str
def _prompt(marker: str) -> str:
return f"synthetic responses request marker-{marker}"
def _deployment(scenario: Scenario, wire: Wire) -> str:
return scenario.model(
model=f"bedrock/{GPT}",
api_key=TOKEN,
aws_region_name="us-east-1",
aws_bedrock_runtime_endpoint=wire.url,
api_base=None,
)
def _issued_id(client_id: str) -> _IssuedId:
decrypted: Final = decrypt_if_encrypted_with(client_id.removeprefix("resp_"), SALT)
assert decrypted is not None, client_id
issued: Final = decrypted.split(";")[0].split("response_id:")[-1]
decoded: Final = base64.b64decode(issued.removeprefix("resp_")).decode()
return _IssuedId(issued, decoded.split(";")[-1].removeprefix("response_id:"))
def _native_request(wire: Wire) -> Request:
received: Final = wire.drain()
assert [(request.method, target_of(request)) for request in received] == [("POST", NATIVE_RESPONSES)], received
assert received[0].headers["authorization"] == f"Bearer {TOKEN}", dict(received[0].headers)
return received[0]
def _body(request: Request) -> dict[str, JsonValue]:
return _JSON_OBJECT.validate_json(request.body)
# TODO: a Bedrock non-stream /v1/responses spend row can carry the pre-encryption resp_<base64> id instead of the
# ciphertext the caller received, because the spend row id is read from response_obj["id"] before the
# ResponsesIDSecurity hook rewrites it in place; the row is looked up under both ids until that ordering is fixed on
# main
def _spend_row(client_id: str, issued_id: str) -> dict[str, JsonValue]:
rows: Final = eventually(
lambda: read_rows(
'SELECT model_group, status, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" '
"WHERE request_id = ANY(%s)",
([client_id, issued_id],), # pyright: ignore[reportArgumentType] # psycopg adapts the list to a text array
),
lambda found: len(found) == 1,
seconds=70,
)
return rows[0]
def _success_row(model: str) -> dict[str, JsonValue]:
return {"model_group": model, "status": "success", "prompt_tokens": 30, "completion_tokens": 5}
def test_openai_sdk_responses_request_is_served_by_the_native_responses_route(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
client: Final = openai.OpenAI(base_url=f"{gateway.client.base_url}/v1", api_key=gateway.key, max_retries=0)
raw: Final = client.responses.with_raw_response.create(
model=model, input=_prompt(marker), extra_body={"cache": {"no-cache": True}}
)
response: Final = raw.parse()
assert response.output_text == answer(marker), raw.text
assert response.usage is not None and (response.usage.input_tokens, response.usage.output_tokens) == (30, 5)
issued: Final = _issued_id(response.id)
assert issued.upstream == f"resp_upstream_{marker}", response.id
request: Final = _native_request(wire)
assert _body(request) == {"model": GPT, "input": _prompt(marker)}, request.body
assert _spend_row(response.id, issued.issued) == _success_row(model)
async def test_async_openai_sdk_responses_stream_is_served_by_the_native_responses_route(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
client: Final = openai.AsyncOpenAI(base_url=f"{gateway.client.base_url}/v1", api_key=gateway.key, max_retries=0)
stream: Final = await client.responses.create(
model=model, input=_prompt(marker), stream=True, extra_body={"cache": {"no-cache": True}}
)
events: Final = [event async for event in stream]
assert [event.type for event in events] == [
"response.created",
"response.output_text.delta",
"response.completed",
], events
deltas: Final = "".join(event.delta for event in events if isinstance(event, ResponseTextDeltaEvent))
assert deltas == answer(marker), events
completed: Final = events[-1]
assert isinstance(completed, ResponseCompletedEvent), completed
assert completed.response.output_text == answer(marker), completed
issued: Final = _issued_id(completed.response.id)
assert issued.upstream == f"resp_upstream_{marker}", completed.response.id
request: Final = _native_request(wire)
assert _body(request) == {"model": GPT, "input": _prompt(marker), "stream": True}, request.body
assert _spend_row(completed.response.id, issued.issued) == _success_row(model)

413
tests/integration/providers/test_bedrock_runtime_chat_completions_chaos.py Normal file
View file

@ -0,0 +1,413 @@
import asyncio
import base64
import binascii
import itertools
import multiprocessing
import os
import re
import signal
import socket
import threading
import uuid
from collections.abc import Callable, Iterator, Mapping
from contextlib import contextmanager
from dataclasses import dataclass
from multiprocessing.process import BaseProcess
from multiprocessing.sharedctypes import Synchronized
from pathlib import Path
from queue import SimpleQueue
from types import MappingProxyType
from typing import Final, Literal
from urllib.parse import urlsplit, urlunsplit
import httpx
import psutil
import pytest
import yaml
from integration._support.bedrock_runtime_peer import MARKER, marker_of, respond, serve_peer
from integration._support.client import Gateway, Scenario, eventually, object_value
from integration._support.database import read_rows
from integration._support.process import owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue, TypeAdapter
BEDROCK_MODEL: Final = "us.openai.gpt-5.6-sol"
TOKEN: Final = "synthetic-bedrock-bearer"
_CONFIG_MODEL: Final = "bedrock-gpt-chat-completions-chaos"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]")
_ENDPOINTS: Final[tuple["Endpoint", ...]] = ("chat", "messages", "responses")
Endpoint = Literal["chat", "messages", "responses"]
@dataclass(frozen=True, slots=True)
class _Call:
endpoint: Endpoint
stream: bool
marker: str
@dataclass(frozen=True, slots=True)
class _Served:
call: _Call
status: int
text: str
call_id: str | None
@dataclass(frozen=True, slots=True)
class _ChildPeer:
process: BaseProcess
received: Synchronized[int]
url: str
def _path(endpoint: Endpoint) -> str:
match endpoint:
case "chat":
return "/v1/chat/completions"
case "messages":
return "/v1/messages"
case "responses":
return "/v1/responses"
def _terminal(endpoint: Endpoint) -> str:
match endpoint:
case "chat":
return "data: [DONE]"
case "messages":
return "event: message_stop"
case "responses":
return '"type":"response.completed"'
def _body(model: str, call: _Call) -> dict[str, JsonValue]:
question: Final = f"Question marker-{call.marker}"
common: Final[dict[str, JsonValue]] = {"model": model, "stream": call.stream, "cache": {"no-cache": True}}
match call.endpoint:
case "chat":
return {**common, "messages": [{"role": "user", "content": question}]}
case "messages":
return {**common, "max_tokens": 64, "messages": [{"role": "user", "content": question}]}
case "responses":
return {**common, "input": question}
def _deployment(scenario: Scenario, endpoint: str) -> str:
return scenario.model(
model=f"bedrock/{BEDROCK_MODEL}",
api_key=TOKEN,
api_base=None,
aws_region_name="us-east-1",
aws_bedrock_runtime_endpoint=endpoint,
)
def _frames(text: str) -> tuple[dict[str, JsonValue], ...]:
return tuple(
_JSON_OBJECT.validate_json(line[6:])
for line in text.splitlines()
if line.startswith("data: ") and line != "data: [DONE]"
)
def _frame_id(frame: Mapping[str, JsonValue]) -> str | None:
if frame.get("type") == "message_start":
return str(object_value(frame["message"])["id"])
response: Final = frame.get("response")
if isinstance(response, dict) and "id" in response:
return str(response["id"])
identity: Final = frame.get("id")
return identity if isinstance(identity, str) else None
def _response_id(served: _Served) -> str:
if not served.call.stream:
return str(_JSON_OBJECT.validate_json(served.text)["id"])
ids: Final = tuple(identity for identity in map(_frame_id, _frames(served.text)) if identity is not None)
assert ids, served.text
return ids[0]
def _assert_answered_with_its_own_marker(served: _Served) -> None:
assert served.status == 200, served.text
assert set(MARKER.findall(served.text)) == {served.call.marker}, served.text
if served.call.stream:
assert _terminal(served.call.endpoint) in served.text, served.text
def _spend_rows(model: str, expected: int) -> list[dict[str, JsonValue]]:
return eventually(
lambda: read_rows('SELECT request_id, status FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)),
lambda found: len(found) >= expected,
seconds=60,
)
def _rows_by_status(rows: list[dict[str, JsonValue]], status: str) -> list[str]:
return sorted(str(row["request_id"]) for row in rows if row["status"] == status)
def _upstream_id_inside(row_id: str) -> str | None:
try:
payload: Final = base64.b64decode(row_id.removeprefix("resp_"), validate=True).decode()
except (binascii.Error, UnicodeDecodeError):
return None
return payload.rsplit("response_id:", 1)[1] if "response_id:" in payload else None
# TODO: a Bedrock non-stream /v1/responses spend row can carry the pre-encryption resp_<base64> id instead of the
# ciphertext the caller received, because the spend row id is read from response_obj["id"] before the
# ResponsesIDSecurity hook rewrites it in place; such a row is matched by the upstream id inside that payload until
# that ordering is fixed on main
def _row_belongs_to(row_id: str, served: _Served) -> bool:
if row_id == _response_id(served):
return True
return served.call.endpoint == "responses" and _upstream_id_inside(row_id) == f"resp_upstream_{served.call.marker}"
def _assert_each_success_landed_once(rows: list[dict[str, JsonValue]], served: tuple[_Served, ...]) -> None:
success_ids: Final = _rows_by_status(rows, "success")
assert len(success_ids) == len(served), rows
for item in served:
owned: Final = [row_id for row_id in success_ids if _row_belongs_to(row_id, item)]
assert len(owned) == 1, (item.call, owned, success_ids)
async def _send(client: httpx.AsyncClient, key: str, model: str, call: _Call) -> _Served:
async with client.stream(
"POST",
_path(call.endpoint),
json=_body(model, call),
headers={"Authorization": f"Bearer {key}", "anthropic-version": "2023-06-01"},
) as response:
raw: Final = await response.aread()
return _Served(
call=call, status=response.status_code, text=raw.decode(), call_id=response.headers.get("x-litellm-call-id")
)
async def _burst(
base_url: str, key: str, model: str, calls: tuple[_Call, ...], *, tolerate_transport_errors: bool = False
) -> tuple[_Served, ...]:
async with httpx.AsyncClient(base_url=base_url, timeout=60, trust_env=False) as client:
results: Final = await asyncio.gather(
*(_send(client, key, model, call) for call in calls), return_exceptions=tolerate_transport_errors
)
for result in results:
assert not isinstance(result, BaseException) or isinstance(result, httpx.TransportError), repr(result)
return tuple(result for result in results if isinstance(result, _Served))
async def _burst_killing_the_peer_once_it_answered(
base_url: str, key: str, model: str, calls: tuple[_Call, ...], peer: _ChildPeer, answered: int
) -> tuple[_Served, ...]:
async with httpx.AsyncClient(base_url=base_url, timeout=60, trust_env=False) as client:
tasks: Final = tuple(asyncio.create_task(_send(client, key, model, call)) for call in calls)
await asyncio.to_thread(eventually, lambda: peer.received.value, lambda count: count == len(calls), 60)
first: Final = [await finished for finished in itertools.islice(asyncio.as_completed(tasks), answered)]
assert all(item.status == 200 for item in first), [(item.call.marker, item.status) for item in first]
peer.process.kill()
peer.process.join(timeout=10)
return tuple(await asyncio.gather(*tasks))
def _calls(count: int, endpoints: tuple[Endpoint, ...], stream: Callable[[int], bool]) -> tuple[_Call, ...]:
return tuple(
_Call(endpoint=endpoints[index % len(endpoints)], stream=stream(index), marker=uuid.uuid4().hex)
for index in range(count)
)
def _free_port() -> int:
with socket.socket() as reserve:
reserve.bind(("127.0.0.1", 0))
return reserve.getsockname()[1]
def _accepts_connections(port: int) -> bool:
try:
with socket.create_connection(("127.0.0.1", port), timeout=0.2):
return True
except OSError:
return False
@contextmanager
def _child_peer(port: int, answer_first: int) -> Iterator[_ChildPeer]:
context: Final = multiprocessing.get_context("spawn")
received: Final = context.Value("i", 0)
process: Final = context.Process(target=serve_peer, args=(port, received, answer_first), daemon=True)
process.start()
try:
eventually(lambda: _accepts_connections(port), bool, seconds=30)
yield _ChildPeer(process=process, received=received, url=f"http://127.0.0.1:{port}")
finally:
process.kill()
process.join(timeout=10)
assert not process.is_alive(), "Owned peer survived cleanup"
async def test_burst_across_every_endpoint_lands_each_response_id_once(gateway: Gateway) -> None:
calls: Final = _calls(36, _ENDPOINTS, lambda index: index % 2 == 0)
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire.url)
served: Final = await _burst(str(gateway.client.base_url), gateway.key, model, calls)
assert len(served) == 36
for item in served:
_assert_answered_with_its_own_marker(item)
ids: Final = sorted(_response_id(item) for item in served)
assert len(set(ids)) == 36, ids
assert sorted(marker_of(request) for request in wire.drain()) == sorted(call.marker for call in calls)
rows: Final = _spend_rows(model, 36)
_assert_each_success_landed_once(rows, served)
assert len(rows) == 36, rows
@pytest.mark.timeout(180)
async def test_peer_killed_mid_burst_fails_only_the_held_calls_and_a_restarted_peer_serves_again(
gateway: Gateway,
) -> None:
calls: Final = _calls(12, _ENDPOINTS, lambda index: index % 2 == 0)
recovery: Final = _calls(6, _ENDPOINTS, lambda index: index % 2 == 1)
port: Final = _free_port()
with gateway.scenario() as scenario:
model: Final = _deployment(scenario, f"http://127.0.0.1:{port}")
with _child_peer(port, answer_first=6) as peer:
served: Final = await _burst_killing_the_peer_once_it_answered(
str(gateway.client.base_url), gateway.key, model, calls, peer, answered=6
)
succeeded: Final = tuple(item for item in served if item.status == 200)
failed: Final = tuple(item for item in served if item.status != 200)
assert (len(succeeded), len(failed)) == (6, 6), [(item.call.marker, item.status) for item in served]
for item in succeeded:
_assert_answered_with_its_own_marker(item)
assert {item.status for item in failed} == {503}, [
(item.call.endpoint, item.call.stream, item.status, item.text) for item in failed
]
for item in failed:
assert "ServiceUnavailableError: BedrockException - Server disconnected" in item.text, item.text
assert "marker-" not in item.text and item.call_id is not None, item.text
with _child_peer(port, answer_first=10**6) as revived:
recovered: Final = await _burst(str(gateway.client.base_url), gateway.key, model, recovery)
assert revived.received.value == 6, revived.received.value
for item in recovered:
_assert_answered_with_its_own_marker(item)
rows: Final = _spend_rows(model, 18)
_assert_each_success_landed_once(rows, (*succeeded, *recovered))
assert _rows_by_status(rows, "failure") == sorted(str(item.call_id) for item in failed), rows
assert len(rows) == 18, rows
async def test_slow_peer_streams_are_forwarded_once_and_terminated(gateway: Gateway) -> None:
calls: Final = _calls(10, ("chat",), lambda _: True)
with wire_server(lambda request: respond(request, pause=0.3)) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire.url)
served: Final = await _burst(str(gateway.client.base_url), gateway.key, model, calls)
assert len(served) == 10
for item in served:
_assert_answered_with_its_own_marker(item)
assert sorted(marker_of(request) for request in wire.drain()) == sorted(call.marker for call in calls)
ids: Final = sorted(_response_id(item) for item in served)
rows: Final = _spend_rows(model, 10)
assert _rows_by_status(rows, "success") == ids, rows
assert len(rows) == 10, rows
def _chaos_config(wire: Wire, tmp_path: Path) -> Path:
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
config["model_list"] = [
{
"model_name": _CONFIG_MODEL,
"litellm_params": {
"model": f"bedrock/{BEDROCK_MODEL}",
"api_key": TOKEN,
"aws_region_name": "us-east-1",
"aws_bedrock_runtime_endpoint": wire.url,
},
}
]
path: Final = tmp_path / "bedrock-gpt-chat-completions-chaos.yaml"
path.write_text(yaml.safe_dump(config))
return path
def _pooled_database_url() -> str:
parts: Final = urlsplit(os.environ["DATABASE_URL"])
query: Final = "&".join(part for part in (parts.query, "connection_limit=5") if part)
return urlunsplit(parts._replace(query=query))
def _open_upstream_connections(pid: int, upstream: str) -> int:
port: Final = urlsplit(upstream).port
return sum(
1
for connection in psutil.Process(pid).net_connections(kind="tcp")
if connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == port
)
def _landed_once(ids: tuple[str, ...]) -> list[dict[str, JsonValue]]:
return eventually(
lambda: read_rows(
'SELECT request_id, status FROM "LiteLLM_SpendLogs" WHERE request_id = ANY(%s)',
(list(ids),), # pyright: ignore[reportArgumentType] # psycopg adapts the list to a text array
),
lambda found: len(found) >= len(ids),
seconds=60,
)
@pytest.mark.timeout(180)
async def test_worker_sigkill_mid_burst_leaves_the_sibling_serving(gateway: Gateway, tmp_path: Path) -> None:
calls: Final = _calls(20, ("chat",), lambda _: False)
release: Final = threading.Event()
held_markers: Final[SimpleQueue[str]] = SimpleQueue()
def held(request: Request) -> Reply:
held_markers.put(marker_of(request))
assert release.wait(timeout=60), "The burst was never released"
return respond(request)
with wire_server(held) as wire:
path: Final = _chaos_config(wire, tmp_path)
overrides: Final = {"DATABASE_URL": _pooled_database_url()}
with owned_proxy_process(gateway, tmp_path, overrides, config=path, workers=2) as owned:
candidate: Final = owned.gateway
workers: Final = eventually(
lambda: tuple(int(pid) for pid in _STARTED_WORKER.findall(owned.log.read_text())),
lambda pids: len(pids) == 2,
seconds=30,
)
burst: Final = asyncio.create_task(
_burst(
str(candidate.client.base_url), candidate.key, _CONFIG_MODEL, calls, tolerate_transport_errors=True
)
)
await asyncio.to_thread(eventually, held_markers.qsize, lambda size: size == 20, 60)
held_by: Final = MappingProxyType({pid: _open_upstream_connections(pid, wire.url) for pid in workers})
assert sum(held_by.values()) == 20, held_by
victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__)
victim: Final = psutil.Process(victim_pid)
victim.suspend()
victim.send_signal(signal.SIGKILL)
release.set()
served: Final = await burst
assert held_by[survivor_pid] >= 10, held_by
assert len(served) == held_by[survivor_pid], (held_by, len(served))
for item in served:
_assert_answered_with_its_own_marker(item)
follow_up: Final = _Call(endpoint="chat", stream=False, marker=uuid.uuid4().hex)
(answered,) = await _burst(str(candidate.client.base_url), candidate.key, _CONFIG_MODEL, (follow_up,))
_assert_answered_with_its_own_marker(answered)
received: Final = wire.drain()
assert {request.method for request in received} == {"POST"}, received
assert sorted(marker_of(request) for request in received) == sorted(
call.marker for call in (*calls, follow_up)
)
ids: Final = tuple(sorted(_response_id(item) for item in (*served, answered)))
rows: Final = _landed_once(ids)
assert _rows_by_status(rows, "success") == list(ids), rows
assert len(rows) == len(ids), rows

437
tests/integration/providers/test_bedrock_runtime_chat_completions_sad_wire.py Normal file
View file

@ -0,0 +1,437 @@
import json
import os
import time
import uuid
from collections.abc import Mapping
from concurrent.futures import ThreadPoolExecutor
from hashlib import sha256
from pathlib import Path
from types import MappingProxyType
from typing import Final
from urllib.parse import urlsplit, urlunsplit
import httpx
import pytest
import yaml
from integration._support.bedrock_runtime_peer import answer, forwarded_effort, marker_of, respond, target_of
from integration._support.client import Gateway, Scenario, eventually, object_value, string_value
from integration._support.database import read_rows
from integration._support.process import owned_proxy_process
from integration._support.wire import Request, Wire, wire_server
from pydantic import JsonValue, TypeAdapter
GPT: Final = "us.openai.gpt-5.6-sol"
TOKEN: Final = "synthetic-bedrock-bearer"
BAD_KEY: Final = "sk-synthetic-bad-key"
NATIVE_TARGET: Final = "/openai/v1/chat/completions"
CONVERSE_TARGET: Final = f"/model/{GPT}/converse"
LONG_VERSION_GPT: Final = "openai.gpt-" + "1" * 30000
PNG_DATA_URL: Final = (
"data:image/png;base64,"
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="
)
GPT_DEPLOYMENT: Final[Mapping[str, JsonValue]] = MappingProxyType(
{"model": f"bedrock/{GPT}", "api_key": TOKEN, "aws_region_name": "us-east-1"}
)
_ALLOWLISTED_MODEL: Final = "bedrock-gpt-image-allowlist"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
def _prompt(marker: str) -> str:
return f"synthetic sad request marker-{marker}"
def _messages(marker: str) -> list[dict[str, JsonValue]]:
return [{"role": "user", "content": _prompt(marker)}]
def _image_messages(marker: str, url: str) -> list[dict[str, JsonValue]]:
return [
{
"role": "user",
"content": [{"type": "text", "text": _prompt(marker)}, {"type": "image_url", "image_url": {"url": url}}],
}
]
def _deployment(scenario: Scenario, wire: Wire, **overrides: JsonValue) -> str:
return scenario.model(**{**GPT_DEPLOYMENT, "aws_bedrock_runtime_endpoint": wire.url, **overrides})
def _chat(gateway: Gateway, model: str, marker: str, *, key: str | None = None, **params: JsonValue) -> httpx.Response:
return gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": _messages(marker), "cache": {"no-cache": True}, **params},
key=key,
)
def _payload(response: httpx.Response) -> dict[str, JsonValue]:
assert response.status_code == 200, response.text
return _JSON_OBJECT.validate_json(response.content)
def _content(response: httpx.Response) -> JsonValue:
choices: Final = _payload(response)["choices"]
assert isinstance(choices, list), response.text
return object_value(object_value(choices[0])["message"])["content"]
def _error_message(response: httpx.Response) -> str:
return string_value(object_value(_JSON_OBJECT.validate_json(response.content)["error"])["message"])
def _call_id(response: httpx.Response) -> str:
return response.headers["x-litellm-call-id"]
def _body(request: Request) -> dict[str, JsonValue]:
return _JSON_OBJECT.validate_json(request.body)
def _routes(received: tuple[Request, ...]) -> list[tuple[str, str]]:
return [(request.method, target_of(request)) for request in received]
def _only_request(wire: Wire, marker: str) -> Request:
received: Final = wire.drain()
assert len(received) == 1, _routes(received)
assert marker_of(received[0]) == marker, received[0].body
return received[0]
def _spend_rows(identity: str) -> list[dict[str, JsonValue]]:
return read_rows(
'SELECT request_id, model_group, status, cache_hit, spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s',
(identity,),
)
def _spend_row(identity: str) -> dict[str, JsonValue]:
return eventually(lambda: _spend_rows(identity), lambda found: len(found) == 1, seconds=70)[0]
def _assert_row(identity: str, model: str, status: str) -> None:
row: Final = _spend_row(identity)
assert (row["model_group"], row["status"]) == (model, status), row
def _timed_liveliness(gateway: Gateway) -> tuple[int, float]:
started: Final = time.monotonic()
response: Final = gateway.request("GET", "/health/liveliness")
return response.status_code, time.monotonic() - started
def _pooled_database_url(url: str) -> str:
parts: Final = urlsplit(url)
query: Final = "&".join(part for part in (parts.query, "connection_limit=5") if part)
return urlunsplit(parts._replace(query=query))
def _allowlist_config(wire: Wire, tmp_path: Path) -> Path:
config: Final = _JSON_OBJECT.validate_python(
yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
)
path: Final = tmp_path / "bedrock-gpt-image-allowlist.yaml"
path.write_text(
yaml.safe_dump(
{
**config,
"model_list": [
{
"model_name": _ALLOWLISTED_MODEL,
"litellm_params": {**GPT_DEPLOYMENT, "aws_bedrock_runtime_endpoint": wire.url},
}
],
"general_settings": {
**object_value(config["general_settings"]),
"user_url_allowed_hosts": ["127.0.0.1"],
},
}
)
)
return path
def test_remote_image_url_on_the_shared_proxy_is_rejected_before_any_fetch(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": _image_messages(marker, f"{wire.url}/image.png"), "cache": {"no-cache": True}},
)
assert response.status_code == 400, response.text
message: Final = _error_message(response)
assert "Unable to fetch image from URL" in message and "user_url_allowed_hosts" in message, response.text
_assert_row(_call_id(response), model, "failure")
assert _routes(wire.drain()) == []
@pytest.mark.timeout(180)
def test_allowlisted_remote_image_is_inlined_for_the_native_route(gateway: Gateway, tmp_path: Path) -> None:
marker: Final = uuid.uuid4().hex
missing_marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire:
path: Final = _allowlist_config(wire, tmp_path)
overrides: Final = {"DATABASE_URL": _pooled_database_url(os.environ["DATABASE_URL"])}
with owned_proxy_process(gateway, tmp_path, overrides, config=path) as owned:
candidate: Final = owned.gateway
response: Final = candidate.request(
"POST",
"/v1/chat/completions",
{
"model": _ALLOWLISTED_MODEL,
"messages": _image_messages(marker, f"{wire.url}/image.png"),
"cache": {"no-cache": True},
},
)
assert _content(response) == answer(marker), response.text
received: Final = wire.drain()
assert _routes(received) == [("GET", "/image.png"), ("POST", NATIVE_TARGET)], received
assert _payload(response)["id"] == f"chatcmpl-{marker}", response.text
assert _body(received[1]) == {
"model": GPT,
"messages": _image_messages(marker, PNG_DATA_URL),
"stream": False,
}, received[1].body
_assert_row(f"chatcmpl-{marker}", _ALLOWLISTED_MODEL, "success")
missing: Final = candidate.request(
"POST",
"/v1/chat/completions",
{
"model": _ALLOWLISTED_MODEL,
"messages": _image_messages(missing_marker, f"{wire.url}/missing.png"),
"cache": {"no-cache": True},
},
)
assert missing.status_code == 400, missing.text
assert "Unable to fetch image from URL. Status code: 404" in _error_message(missing), missing.text
_assert_row(_call_id(missing), _ALLOWLISTED_MODEL, "failure")
assert _routes(wire.drain()) == [("GET", "/missing.png")]
def test_response_cache_twin_serves_the_second_request_without_a_second_wire_call(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
body: Final[dict[str, JsonValue]] = {"model": model, "messages": _messages(marker)}
first: Final = gateway.request("POST", "/v1/chat/completions", body)
second: Final = gateway.request("POST", "/v1/chat/completions", body)
identity: Final = string_value(_payload(first)["id"])
assert _content(first) == answer(marker), first.text
assert _payload(second)["id"] == identity, (first.text, second.text)
assert _content(second) == answer(marker), second.text
_only_request(wire, marker)
rows: Final = eventually(
lambda: read_rows(
'SELECT request_id, cache_hit, spend FROM "LiteLLM_SpendLogs" WHERE starts_with(request_id, %s)'
" ORDER BY request_id",
(identity,),
),
lambda found: len(found) == 2,
seconds=70,
)
assert [(row["request_id"] == identity, row["cache_hit"]) for row in rows] == [(True, "None"), (False, "True")]
assert string_value(rows[1]["request_id"]).startswith(f"{identity}_cache_hit"), rows
assert rows[1]["spend"] == 0.0, rows
assert isinstance(rows[0]["spend"], float) and rows[0]["spend"] > 0.0, rows
def test_model_group_info_lists_the_native_supported_params(gateway: Gateway) -> None:
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
groups: Final = gateway.get("/model_group/info", {"model_group": model})["data"]
assert isinstance(groups, list) and len(groups) == 1, groups
group: Final = object_value(groups[0])
assert group["model_group"] == model, group
params: Final = group["supported_openai_params"]
assert isinstance(params, list), group
assert {"reasoning_effort", "logprobs", "top_logprobs"} <= set(params) and "n" not in params, params
assert _routes(wire.drain()) == []
def test_thirty_thousand_digit_version_is_classified_quickly_and_served_by_converse(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario, ThreadPoolExecutor(max_workers=1) as pool:
model: Final = _deployment(scenario, wire, model=f"bedrock/{LONG_VERSION_GPT}")
liveliness: Final = pool.submit(_timed_liveliness, gateway)
started: Final = time.monotonic()
response: Final = _chat(gateway, model, marker)
elapsed: Final = time.monotonic() - started
health_status, health_elapsed = liveliness.result()
assert _content(response) == answer(marker), response.text
assert elapsed < 10, elapsed
assert (health_status, health_elapsed < 2) == (200, True), (health_status, health_elapsed)
request: Final = _only_request(wire, marker)
assert (request.method, target_of(request)) == ("POST", f"/model/{LONG_VERSION_GPT}/converse"), request.target
_assert_row(string_value(_payload(response)["id"]), model, "success")
def test_bad_key_on_the_long_version_model_is_refused_before_any_route(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
control_marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire, model=f"bedrock/{LONG_VERSION_GPT}")
started: Final = time.monotonic()
refused: Final = _chat(gateway, model, marker, key=BAD_KEY)
elapsed: Final = time.monotonic() - started
assert refused.status_code == 401, refused.text
assert elapsed < 2, elapsed
assert "Authentication Error" in _error_message(refused), refused.text
refused_rows: Final = eventually(
lambda: read_rows(
"SELECT request_id, status, spend, metadata->'error_information'->>'error_code' AS error_code"
' FROM "LiteLLM_SpendLogs" WHERE model_group=%s AND api_key=%s',
(model, sha256(BAD_KEY.encode()).hexdigest()),
),
lambda found: len(found) == 1,
seconds=70,
)
assert (refused_rows[0]["status"], refused_rows[0]["spend"], refused_rows[0]["error_code"]) == (
"failure",
0.0,
"401",
), refused_rows
control: Final = _chat(gateway, model, control_marker)
control_id: Final = string_value(_payload(control)["id"])
_assert_row(control_id, model, "success")
landed: Final = read_rows('SELECT request_id FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,))
assert {row["request_id"] for row in landed} == {control_id, refused_rows[0]["request_id"]}, landed
received: Final = wire.drain()
assert [marker_of(request) for request in received] == [control_marker], _routes(received)
@pytest.mark.parametrize("effort", [pytest.param("", id="empty"), pytest.param("x" * 5120, id="five_kb")])
def test_invalid_reasoning_effort_reaches_the_peer_and_its_400_reaches_the_caller(
gateway: Gateway, effort: str
) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, reasoning_effort=effort)
assert response.status_code == 400, response.text
peer_error: Final = json.dumps({"message": f"Invalid reasoning effort: {json.dumps(effort)}"})
assert f"BedrockException - {peer_error}" in _error_message(response), response.text
request: Final = _only_request(wire, marker)
assert forwarded_effort(request) == effort, request.body
_assert_row(_call_id(response), model, "failure")
NON_STRING_EFFORTS: Final = (pytest.param(7, id="int"), pytest.param(["high"], id="list"))
@pytest.mark.parametrize("effort", NON_STRING_EFFORTS)
def test_non_string_reasoning_effort_is_refused_before_any_wire_request(gateway: Gateway, effort: JsonValue) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, reasoning_effort=effort)
assert response.status_code == 400, response.text
message: Final = _error_message(response)
assert message.startswith("litellm.UnsupportedParamsError"), response.text
assert "reasoning_effort as a string" in message and "drop_params" in message, response.text
_assert_row(_call_id(response), model, "failure")
assert _routes(wire.drain()) == []
@pytest.mark.parametrize("effort", NON_STRING_EFFORTS)
def test_drop_params_deployment_drops_a_non_string_reasoning_effort(gateway: Gateway, effort: JsonValue) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire, drop_params=True)
response: Final = _chat(gateway, model, marker, reasoning_effort=effort)
assert _content(response) == answer(marker), response.text
request: Final = _only_request(wire, marker)
assert target_of(request) == NATIVE_TARGET, request.body
assert "reasoning_effort" not in _body(request), request.body
_assert_row(string_value(_payload(response)["id"]), model, "success")
def test_duplicated_reasoning_effort_key_lets_the_last_value_win(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
prefix: Final = json.dumps({"model": model, "messages": _messages(marker), "cache": {"no-cache": True}})[:-1]
response: Final = gateway.client.post(
"/v1/chat/completions",
content=f'{prefix}, "reasoning_effort": "low", "reasoning_effort": "high"}}'.encode(),
headers={"Authorization": f"Bearer {gateway.key}", "content-type": "application/json"},
)
assert _content(response) == answer(marker), response.text
request: Final = _only_request(wire, marker)
assert forwarded_effort(request) == "high", request.body
_assert_row(string_value(_payload(response)["id"]), model, "success")
def test_string_temperature_is_refused_before_any_wire_request(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, temperature="0.2")
assert response.status_code == 400, response.text
message: Final = _error_message(response)
assert message.startswith("litellm.UnsupportedParamsError") and "['temperature']" in message, response.text
_assert_row(_call_id(response), model, "failure")
assert _routes(wire.drain()) == []
@pytest.mark.parametrize(
("scripted", "expected"),
[pytest.param(401, 401, id="401"), pytest.param(429, 429, id="429"), pytest.param(500, 503, id="500")],
)
def test_peer_error_status_reaches_the_caller_and_unrelated_deployments_keep_serving(
gateway: Gateway, scripted: int, expected: int
) -> None:
marker: Final = uuid.uuid4().hex
control_marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
unrelated: Final = scenario.model()
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [{"role": "user", "content": f"status={scripted} marker-{marker}"}],
"cache": {"no-cache": True},
},
)
assert response.status_code == expected, response.text
assert f'BedrockException - {{"message": "scripted {scripted}"}}' in _error_message(response), response.text
_only_request(wire, marker)
_assert_row(_call_id(response), model, "failure")
control: Final = _chat(gateway, unrelated, control_marker)
assert control.status_code == 200, control.text
_assert_row(string_value(_payload(control)["id"]), unrelated, "success")
assert _routes(wire.drain()) == []
@pytest.mark.parametrize(
"params", [pytest.param({"reasoning_effort": None}, id="null"), pytest.param({}, id="missing")]
)
def test_absent_reasoning_effort_is_forwarded_as_absent_on_every_repeat(
gateway: Gateway, params: dict[str, JsonValue]
) -> None:
markers: Final = tuple(uuid.uuid4().hex for _ in range(3))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
responses: Final = tuple(_chat(gateway, model, marker, **params) for marker in markers)
assert [_content(response) for response in responses] == [answer(marker) for marker in markers]
ids: Final = tuple(string_value(_payload(response)["id"]) for response in responses)
assert len(set(ids)) == 3, ids
received: Final = wire.drain()
assert [marker_of(request) for request in received] == list(markers), _routes(received)
assert [forwarded_effort(request) for request in received] == [None, None, None], [_body(r) for r in received]
rows: Final = eventually(
lambda: read_rows(
'SELECT request_id, status FROM "LiteLLM_SpendLogs" WHERE request_id IN (%s, %s, %s)', ids
),
lambda found: len(found) == 3,
seconds=70,
)
assert {(string_value(row["request_id"]), row["status"]) for row in rows} == {
(identity, "success") for identity in ids
}, rows

549
tests/integration/providers/test_bedrock_runtime_chat_completions_wire.py Normal file
View file

@ -0,0 +1,549 @@
import json
import uuid
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final
from urllib.parse import quote
import httpx
import openai
import pytest
from integration._support.bedrock_runtime_peer import answer, respond, target_of
from integration._support.client import Gateway, Scenario, eventually
from integration._support.database import read_rows
from integration._support.sigv4 import signature
from integration._support.wire import Request, Wire, wire_server
from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
from openai.types.chat.chat_completion_chunk import ChoiceDelta
from pydantic import JsonValue, TypeAdapter
GPT: Final = "us.openai.gpt-5.6-sol"
GLOBAL_GPT: Final = "global.openai.gpt-5.6-sol"
GPT_OSS: Final = "openai.gpt-oss-120b-1:0"
TOKEN: Final = "synthetic-bedrock-bearer"
ACCESS_KEY: Final = "AKIASYNTHETICKEY0001"
SECRET_KEY: Final = "synthetic-secret-key-for-testing"
PROFILE_ARN: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/a1b2c3d4e5f6"
NATIVE_TARGET: Final = "/openai/v1/chat/completions"
CONVERSE_TARGET: Final = f"/model/{GPT}/converse"
GPT_DEPLOYMENT: Final[Mapping[str, JsonValue]] = MappingProxyType(
{"model": f"bedrock/{GPT}", "api_key": TOKEN, "aws_region_name": "us-east-1"}
)
GUARDRAIL: Final[Mapping[str, JsonValue]] = MappingProxyType(
{"guardrailIdentifier": "gr-synthetic", "guardrailVersion": "1"}
)
TOOL_PARAMETERS: Final[Mapping[str, JsonValue]] = MappingProxyType(
{"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}
)
TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType(
{
"type": "function",
"function": {
"name": "lookup_invoice",
"description": "Look up an invoice",
"parameters": dict(TOOL_PARAMETERS),
},
}
)
CONVERSE_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType(
{
"toolSpec": {
"inputSchema": {"json": dict(TOOL_PARAMETERS)},
"name": "lookup_invoice",
"description": "Look up an invoice",
}
}
)
JSON_SCHEMA: Final[Mapping[str, JsonValue]] = MappingProxyType(
{
"type": "json_schema",
"json_schema": {
"name": "verdict",
"strict": True,
"schema": {
"type": "object",
"properties": {"ok": {"type": "boolean"}},
"required": ["ok"],
"additionalProperties": False,
},
},
}
)
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_OBSERVATIONS: Final = TypeAdapter(list[dict[str, JsonValue]])
def _prompt(marker: str) -> str:
return f"synthetic native request marker-{marker}"
def _messages(marker: str) -> list[JsonValue]:
return [{"role": "user", "content": _prompt(marker)}]
def _sdk_messages(marker: str) -> list[ChatCompletionMessageParam]:
return [{"role": "user", "content": _prompt(marker)}]
def _converse_messages(marker: str) -> list[JsonValue]:
return [{"role": "user", "content": [{"text": _prompt(marker)}]}]
def _native_body(model: str, marker: str, **params: JsonValue) -> dict[str, JsonValue]:
return {"model": model, "messages": _messages(marker), "stream": False, **params}
def _streamed_native_body(model: str, marker: str) -> dict[str, JsonValue]:
return _native_body(model, marker, stream=True, stream_options={"include_usage": True})
def _deployment(scenario: Scenario, wire: Wire, **overrides: JsonValue) -> str:
return scenario.model(model_info=None, **{**GPT_DEPLOYMENT, "aws_bedrock_runtime_endpoint": wire.url, **overrides})
def _openai_client(gateway: Gateway) -> openai.OpenAI:
return openai.OpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0)
def _async_openai_client(gateway: Gateway) -> openai.AsyncOpenAI:
return openai.AsyncOpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0)
def _chat(gateway: Gateway, model: str, marker: str, **params: JsonValue) -> httpx.Response:
return gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": _messages(marker), "cache": {"no-cache": True}, **params},
)
def _payload(response: httpx.Response) -> dict[str, JsonValue]:
assert response.status_code == 200, response.text
return _JSON_OBJECT.validate_json(response.content)
def _only_request(wire: Wire) -> Request:
received: Final = wire.drain()
assert len(received) == 1, [(request.method, target_of(request)) for request in received]
return received[0]
def _body(request: Request) -> dict[str, JsonValue]:
return _JSON_OBJECT.validate_json(request.body)
def _native_request(wire: Wire) -> Request:
request: Final = _only_request(wire)
assert (request.method, target_of(request)) == ("POST", NATIVE_TARGET), request.target
assert request.headers["authorization"] == f"Bearer {TOKEN}", dict(request.headers)
return request
def _converse_request(wire: Wire, target: str = CONVERSE_TARGET) -> Request:
request: Final = _only_request(wire)
assert (request.method, target_of(request)) == ("POST", target), request.target
assert request.headers["authorization"] == f"Bearer {TOKEN}", dict(request.headers)
return request
def _spend_row(identity: str) -> dict[str, JsonValue]:
rows: Final = eventually(
lambda: read_rows(
'SELECT model_group, status, prompt_tokens, completion_tokens, api_base FROM "LiteLLM_SpendLogs"'
" WHERE request_id=%s",
(identity,),
),
lambda found: len(found) == 1,
seconds=70,
)
return rows[0]
def _success_row(model: str, api_base: str) -> dict[str, JsonValue]:
return {"model_group": model, "status": "success", "prompt_tokens": 9, "completion_tokens": 5, "api_base": api_base}
def _delta_text(delta: ChoiceDelta, field: str) -> str:
value: Final = delta.model_dump().get(field)
return value if isinstance(value, str) else ""
def _chunk_text(chunk: ChatCompletionChunk, field: str) -> str:
return "".join(_delta_text(choice.delta, field) for choice in chunk.choices)
def _joined(chunks: Sequence[ChatCompletionChunk], field: str) -> str:
return "".join(_chunk_text(chunk, field) for chunk in chunks)
def _upstream_requests_mentioning(gateway: Gateway, marker: str) -> list[dict[str, JsonValue]]:
observed: Final = httpx.get(f"{gateway.upstream_url}/__observations", trust_env=False, timeout=15)
observed.raise_for_status()
requests: Final = _OBSERVATIONS.validate_python(_JSON_OBJECT.validate_json(observed.content)["requests"])
return [request for request in requests if marker in json.dumps(request["body"])]
def _authorization_field(part: str) -> tuple[str, str]:
name, _, value = part.partition("=")
return name, value
def _assert_sigv4_signed(request: Request, path: str) -> None:
authorization: Final = request.headers["authorization"]
assert authorization.startswith("AWS4-HMAC-SHA256 "), dict(request.headers)
fields: Final = dict(
_authorization_field(part) for part in authorization.removeprefix("AWS4-HMAC-SHA256 ").split(", ")
)
access_key, scope = fields["Credential"].split("/", 1)
assert access_key == ACCESS_KEY, authorization
assert scope == f"{request.headers['x-amz-date'][:8]}/us-east-1/bedrock/aws4_request", authorization
assert {"host", "x-amz-date"}.issubset(fields["SignedHeaders"].split(";")), authorization
expected: Final = signature("POST", path, request.headers, fields["SignedHeaders"], request.body, SECRET_KEY, scope)
assert fields["Signature"] == expected[1], authorization
def test_openai_sdk_reasoning_request_is_served_by_native_chat_completions(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
raw: Final = _openai_client(gateway).chat.completions.with_raw_response.create(
model=model,
messages=_sdk_messages(marker),
reasoning_effort="high",
max_tokens=16,
extra_body={"cache": {"no-cache": True}},
)
completion: Final = raw.parse()
assert completion.id == f"chatcmpl-{marker}", raw.text
assert completion.choices[0].message.content == answer(marker), raw.text
assert completion.usage is not None and completion.usage.model_dump(exclude_none=True) == {
"prompt_tokens": 9,
"completion_tokens": 5,
"total_tokens": 14,
"completion_tokens_details": {"reasoning_tokens": 3},
}, raw.text
assert raw.headers["llm_provider-x-amzn-requestid"] == marker, dict(raw.headers)
request: Final = _native_request(wire)
assert _body(request) == _native_body(GPT, marker, max_completion_tokens=16, reasoning_effort="high")
assert _spend_row(completion.id) == _success_row(model, f"{wire.url}{NATIVE_TARGET}")
async def test_async_openai_sdk_stream_keeps_the_upstream_id_and_usage(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
identity: Final = f"chatcmpl-{marker}"
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
stream: Final = await _async_openai_client(gateway).chat.completions.create(
model=model,
messages=_sdk_messages(marker),
stream=True,
stream_options={"include_usage": True},
extra_body={"cache": {"no-cache": True}},
)
chunks: Final = [chunk async for chunk in stream]
assert {chunk.id for chunk in chunks} == {identity}, chunks
assert _joined(chunks, "content") == answer(marker), chunks
usage: Final = chunks[-1].usage
assert usage is not None and (usage.prompt_tokens, usage.completion_tokens) == (9, 5), chunks[-1]
assert usage.completion_tokens_details is not None and usage.completion_tokens_details.reasoning_tokens == 3
assert all(chunk.usage is None for chunk in chunks[:-1]), chunks
assert _body(_native_request(wire)) == _streamed_native_body(GPT, marker)
assert _spend_row(identity) == _success_row(model, f"{wire.url}{NATIVE_TARGET}")
def test_temperature_is_forwarded_natively_when_reasoning_is_off(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, temperature=0.2, reasoning_effort="none")
payload: Final = _payload(response)
assert payload["id"] == f"chatcmpl-{marker}", response.text
assert _body(_native_request(wire)) == _native_body(GPT, marker, temperature=0.2, reasoning_effort="none")
assert _spend_row(f"chatcmpl-{marker}") == _success_row(model, f"{wire.url}{NATIVE_TARGET}")
def test_temperature_while_reasoning_is_refused_before_any_wire_request(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, temperature=0.2, reasoning_effort="high")
assert response.status_code == 400, response.text
assert "UnsupportedParamsError" in response.text and "'temperature'" in response.text, response.text
assert wire.drain() == (), response.text
row: Final = _spend_row(response.headers["x-litellm-call-id"])
assert (row["status"], row["model_group"], row["prompt_tokens"]) == ("failure", model, 0), row
assert "while reasoning is active" in response.text, response.text
def test_drop_params_deployment_drops_temperature_while_reasoning(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire, drop_params=True)
response: Final = _chat(gateway, model, marker, temperature=0.2, reasoning_effort="high")
assert _payload(response)["id"] == f"chatcmpl-{marker}", response.text
assert _body(_native_request(wire)) == _native_body(GPT, marker, reasoning_effort="high")
assert _spend_row(f"chatcmpl-{marker}") == _success_row(model, f"{wire.url}{NATIVE_TARGET}")
def test_guardrail_config_keeps_converse(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, guardrailConfig=dict(GUARDRAIL))
payload: Final = _payload(response)
assert payload["choices"] == [
{"finish_reason": "stop", "index": 0, "message": {"content": answer(marker), "role": "assistant"}}
], response.text
assert response.headers["llm_provider-x-amzn-requestid"] == marker, dict(response.headers)
body: Final = _body(_converse_request(wire))
assert body["guardrailConfig"] == GUARDRAIL, body
assert body["messages"] == [
{"role": "user", "content": [{"guardContent": {"text": {"text": _prompt(marker)}}}]}
], body
assert _spend_row(str(payload["id"])) == _success_row(model, f"{wire.url}{CONVERSE_TARGET}")
def test_converse_prefix_pins_the_model_to_converse(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire, model=f"bedrock/converse/{GPT}")
response: Final = _chat(gateway, model, marker, reasoning_effort="high")
payload: Final = _payload(response)
assert payload["choices"] == [
{"finish_reason": "stop", "index": 0, "message": {"content": answer(marker), "role": "assistant"}}
], response.text
body: Final = _body(_converse_request(wire))
assert body["messages"] == _converse_messages(marker), body
assert body["additionalModelRequestFields"] == {"reasoning": {"effort": "high"}}, body
assert _spend_row(str(payload["id"])) == _success_row(model, f"{wire.url}{CONVERSE_TARGET}")
def test_application_inference_profile_arn_keeps_converse(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire, model=f"bedrock/{PROFILE_ARN}")
response: Final = _chat(gateway, model, marker)
payload: Final = _payload(response)
assert payload["choices"] == [
{"finish_reason": "stop", "index": 0, "message": {"content": answer(marker), "role": "assistant"}}
], response.text
request: Final = _converse_request(wire, f"/model/{PROFILE_ARN}/converse")
assert request.target == f"/model/{quote(PROFILE_ARN, safe='')}/converse", request.target
assert _body(request)["messages"] == _converse_messages(marker), request.body
assert _spend_row(str(payload["id"])) == _success_row(
model, f"{wire.url}/model/{quote(PROFILE_ARN, safe='')}/converse"
)
def test_model_id_application_inference_profile_keeps_converse_at_the_profile_url(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire, model_id=PROFILE_ARN)
response: Final = _chat(gateway, model, marker)
payload: Final = _payload(response)
assert payload["choices"] == [
{"finish_reason": "stop", "index": 0, "message": {"content": answer(marker), "role": "assistant"}}
], response.text
request: Final = _converse_request(wire, f"/model/{PROFILE_ARN}/converse")
assert request.target == f"/model/{quote(PROFILE_ARN, safe='')}/converse", request.target
body: Final = _body(request)
assert body["messages"] == _converse_messages(marker), request.body
assert "model_id" not in body and "model" not in body, request.body
assert _spend_row(str(payload["id"])) == _success_row(
model, f"{wire.url}/model/{quote(PROFILE_ARN, safe='')}/converse"
)
def test_stop_sequences_keep_converse(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, stop=["END"])
payload: Final = _payload(response)
assert payload["choices"] == [
{"finish_reason": "stop", "index": 0, "message": {"content": answer(marker), "role": "assistant"}}
], response.text
body: Final = _body(_converse_request(wire))
assert body["messages"] == _converse_messages(marker), body
assert body["inferenceConfig"] == {"stopSequences": ["END"]}, body
assert _spend_row(str(payload["id"])) == _success_row(model, f"{wire.url}{CONVERSE_TARGET}")
def test_json_object_response_format_keeps_converse(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, response_format={"type": "json_object"})
payload: Final = _payload(response)
assert payload["choices"] == [
{"finish_reason": "stop", "index": 0, "message": {"content": answer(marker), "role": "assistant"}}
], response.text
assert _body(_converse_request(wire))["messages"] == _converse_messages(marker), response.text
assert _spend_row(str(payload["id"])) == _success_row(model, f"{wire.url}{CONVERSE_TARGET}")
def test_json_schema_response_format_is_forwarded_natively(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, response_format=dict(JSON_SCHEMA))
assert _payload(response)["id"] == f"chatcmpl-{marker}", response.text
assert _body(_native_request(wire)) == _native_body(GPT, marker, response_format=dict(JSON_SCHEMA))
assert _spend_row(f"chatcmpl-{marker}") == _success_row(model, f"{wire.url}{NATIVE_TARGET}")
def test_tools_while_reasoning_keep_converse(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="high")
payload: Final = _payload(response)
assert payload["choices"] == [
{"finish_reason": "stop", "index": 0, "message": {"content": answer(marker), "role": "assistant"}}
], response.text
body: Final = _body(_converse_request(wire))
assert body["toolConfig"] == {"tools": [CONVERSE_TOOL]}, body
assert body["additionalModelRequestFields"] == {"reasoning": {"effort": "high"}}, body
assert _spend_row(str(payload["id"])) == _success_row(model, f"{wire.url}{CONVERSE_TARGET}")
def test_tools_with_reasoning_off_are_forwarded_natively(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, tools=[dict(TOOL)], reasoning_effort="none")
assert _payload(response)["id"] == f"chatcmpl-{marker}", response.text
assert _body(_native_request(wire)) == _native_body(GPT, marker, tools=[dict(TOOL)], reasoning_effort="none")
assert _spend_row(f"chatcmpl-{marker}") == _success_row(model, f"{wire.url}{NATIVE_TARGET}")
def test_empty_tools_list_while_reasoning_stays_native(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker, tools=[], reasoning_effort="high")
assert _payload(response)["id"] == f"chatcmpl-{marker}", response.text
assert _body(_native_request(wire)) == _native_body(GPT, marker, tools=[], reasoning_effort="high")
assert _spend_row(f"chatcmpl-{marker}") == _success_row(model, f"{wire.url}{NATIVE_TARGET}")
def test_chat_completions_prefix_splits_gpt_oss_reasoning_tag(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire, model=f"bedrock/chat_completions/{GPT_OSS}")
raw: Final = _openai_client(gateway).chat.completions.with_raw_response.create(
model=model, messages=_sdk_messages(marker), extra_body={"cache": {"no-cache": True}}
)
completion: Final = raw.parse()
assert completion.id == f"chatcmpl-{marker}", raw.text
message: Final = completion.choices[0].message
assert message.content == answer(marker), raw.text
assert (message.model_extra or {}).get("reasoning_content") == f"why marker-{marker}", raw.text
assert _body(_native_request(wire)) == _native_body(GPT_OSS, marker)
assert _spend_row(completion.id) == _success_row(model, f"{wire.url}{NATIVE_TARGET}")
def test_chat_completions_prefix_splits_gpt_oss_reasoning_tag_across_stream_deltas(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
identity: Final = f"chatcmpl-{marker}"
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire, model=f"bedrock/chat_completions/{GPT_OSS}")
stream: Final = _openai_client(gateway).chat.completions.create(
model=model,
messages=_sdk_messages(marker),
stream=True,
stream_options={"include_usage": True},
extra_body={"cache": {"no-cache": True}},
)
chunks: Final = list(stream)
assert {chunk.id for chunk in chunks} == {identity}, chunks
assert _joined(chunks, "reasoning_content") == f"why marker-{marker}", chunks
assert _joined(chunks, "content") == answer(marker), chunks
assert _body(_native_request(wire)) == _streamed_native_body(GPT_OSS, marker)
assert _spend_row(identity) == _success_row(model, f"{wire.url}{NATIVE_TARGET}")
def test_region_path_model_is_served_natively_without_the_region(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"bedrock/us-west-2/{GLOBAL_GPT}", api_key=TOKEN, aws_bedrock_runtime_endpoint=wire.url
)
response: Final = _chat(gateway, model, marker)
assert _payload(response)["id"] == f"chatcmpl-{marker}", response.text
assert _body(_native_request(wire)) == _native_body(GLOBAL_GPT, marker)
assert _spend_row(f"chatcmpl-{marker}") == _success_row(model, f"{wire.url}{NATIVE_TARGET}")
def test_sigv4_deployment_signs_the_native_request(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"bedrock/{GPT}",
api_key=None,
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
aws_region_name="us-east-1",
aws_bedrock_runtime_endpoint=wire.url,
)
response: Final = _chat(gateway, model, marker)
assert _payload(response)["id"] == f"chatcmpl-{marker}", response.text
request: Final = _only_request(wire)
assert (request.method, target_of(request)) == ("POST", NATIVE_TARGET), request.target
_assert_sigv4_signed(request, NATIVE_TARGET)
assert _body(request) == _native_body(GPT, marker)
assert _spend_row(f"chatcmpl-{marker}") == _success_row(model, f"{wire.url}{NATIVE_TARGET}")
def test_blank_api_key_on_a_sigv4_deployment_is_signed_not_sent_as_an_empty_bearer(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"bedrock/{GPT}",
api_key="",
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
aws_region_name="us-east-1",
aws_bedrock_runtime_endpoint=wire.url,
)
response: Final = _chat(gateway, model, marker)
assert _payload(response)["id"] == f"chatcmpl-{marker}", response.text
request: Final = _only_request(wire)
assert (request.method, target_of(request)) == ("POST", NATIVE_TARGET), request.target
_assert_sigv4_signed(request, NATIVE_TARGET)
assert _body(request) == _native_body(GPT, marker)
assert _spend_row(f"chatcmpl-{marker}") == _success_row(model, f"{wire.url}{NATIVE_TARGET}")
def test_runtime_endpoint_without_api_base_is_used_natively(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire, api_base=None)
response: Final = _chat(gateway, model, marker)
assert _payload(response)["id"] == f"chatcmpl-{marker}", response.text
assert _body(_native_request(wire)) == _native_body(GPT, marker)
assert _spend_row(f"chatcmpl-{marker}") == _success_row(model, f"{wire.url}{NATIVE_TARGET}")
def test_runtime_endpoint_wins_over_an_unrelated_api_base(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
response: Final = _chat(gateway, model, marker)
assert _payload(response)["id"] == f"chatcmpl-{marker}", response.text
assert _body(_native_request(wire)) == _native_body(GPT, marker)
assert _upstream_requests_mentioning(gateway, marker) == [], response.text
assert _spend_row(f"chatcmpl-{marker}") == _success_row(model, f"{wire.url}{NATIVE_TARGET}")
@pytest.mark.parametrize("suffix", ["/openai/v1", "/openai/v1/chat/completions"])
def test_api_base_already_naming_the_native_path_is_not_doubled(gateway: Gateway, suffix: str) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model_info=None, **{**GPT_DEPLOYMENT, "api_base": f"{wire.url}{suffix}"})
response: Final = _chat(gateway, model, marker)
request: Final = _only_request(wire)
assert (request.method, request.target) == ("POST", NATIVE_TARGET), response.text
assert _payload(response)["id"] == f"chatcmpl-{marker}", response.text
assert _body(request) == _native_body(GPT, marker)
assert _spend_row(f"chatcmpl-{marker}") == _success_row(model, f"{wire.url}{NATIVE_TARGET}")

94
tests/unit/litellm_core_utils/test_image_handling.py
View file

@ -1,4 +1,5 @@
import asyncio
import base64
import copy
import time
import uuid
@ -16,6 +17,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_convert_url_to_base64,
async_inline_remote_media,
convert_url_to_base64,
inline_remote_media,
)
from litellm.litellm_core_utils.url_utils import SSRFError
@ -258,6 +260,54 @@ async def test_async_data_url_is_returned_unchanged_without_fetch(monkeypatch):
assert await async_convert_url_to_base64(data_url) == data_url
REAL_PNG_BYTES = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="
)
def _stub_image_client(content, content_type):
class _Client:
def get(self, url, follow_redirects=True):
headers = {} if content_type is None else {"Content-Type": content_type}
return Response(200, content=content, headers=headers, request=Request("GET", url))
return _Client()
def test_convert_url_to_base64_infers_the_type_when_the_server_sends_octet_stream(monkeypatch):
monkeypatch.setattr(
litellm, "module_level_client", _stub_image_client(REAL_PNG_BYTES, "application/octet-stream")
)
result = convert_url_to_base64(f"http://img.example/{uuid.uuid4()}")
assert result.startswith("data:image/png;base64,")
def test_convert_url_to_base64_keeps_a_real_content_type(monkeypatch):
monkeypatch.setattr(
litellm, "module_level_client", _stub_image_client(REAL_PNG_BYTES, "image/jpeg")
)
result = convert_url_to_base64(f"http://img.example/{uuid.uuid4()}.png")
assert result.startswith("data:image/jpeg;base64,")
def test_convert_url_to_base64_raises_when_no_content_type_is_determinable(monkeypatch):
monkeypatch.setattr(
litellm,
"module_level_client",
_stub_image_client(b"\x00\x01\x02\x03not-an-image", "application/octet-stream"),
)
url = f"http://img.example/{uuid.uuid4()}"
with pytest.raises(litellm.ImageFetchError) as excinfo:
convert_url_to_base64(url)
assert url in str(excinfo.value)
def test_image_size_limit_disabled(monkeypatch):
"""
Test that setting MAX_IMAGE_URL_DOWNLOAD_SIZE_MB to 0 disables all image URL downloads.
@ -320,6 +370,50 @@ async def test_async_inline_remote_media_inlines_every_remote_part_shape(async_o
assert messages == snapshot
def test_inline_remote_media_inlines_every_remote_part_shape(monkeypatch):
image_url = f"http://img.example/{uuid.uuid4()}.png"
pdf_url = f"http://docs.example/{uuid.uuid4()}.pdf"
fetched = []
def fake_convert(url):
fetched.append(url)
return f"data:image/png;base64,{url}"
monkeypatch.setattr(image_handling, "convert_url_to_base64", fake_convert)
messages = [
{"role": "system", "content": "be terse"},
{
"role": "user",
"content": [
{"type": "text", "text": "what is this?"},
{"type": "image_url", "image_url": {"url": image_url, "detail": "low"}},
{"type": "image_url", "image_url": image_url},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}},
{"type": "image_url", "image_url": {"url": "s3://bucket/key.png"}},
{"type": "file", "file": {"file_id": pdf_url}},
{"type": "document", "source": {"type": "url", "url": pdf_url}, "title": "the doc"},
],
},
]
snapshot = copy.deepcopy(messages)
inlined = inline_remote_media(messages, should_inline=image_handling.inline_remote_image_urls)
data_url = f"data:image/png;base64,{image_url}"
assert inlined[0] == {"role": "system", "content": "be terse"}
assert inlined[1]["content"] == [
{"type": "text", "text": "what is this?"},
{"type": "image_url", "image_url": {"url": data_url, "detail": "low"}},
{"type": "image_url", "image_url": data_url},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}},
{"type": "image_url", "image_url": {"url": "s3://bucket/key.png"}},
{"type": "file", "file": {"file_id": pdf_url}},
{"type": "document", "source": {"type": "url", "url": pdf_url}, "title": "the doc"},
]
assert fetched == [image_url]
assert messages == snapshot
async def test_async_inline_remote_media_inlines_only_the_parts_the_predicate_accepts(async_only_image_fetch):
files_api_prefix = "https://generativelanguage.googleapis.com/v1beta/files/"
files_api_pdf = f"{files_api_prefix}{uuid.uuid4().hex}"

0
tests/unit/llms/bedrock/chat/chat_completions/__init__.py Normal file
View file

1469
tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py Normal file
View file

File diff suppressed because it is too large Load diff

21
tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py
View file

@ -162,6 +162,27 @@ class TestForModelGate:
):
assert BedrockOpenAIResponsesConfig.for_model(None) is None
def test_chat_completions_route_keeps_the_native_responses_surface(self):
with patch.object( # test-quality-ok: the gate reads the global cost map by design; no injection point exists
litellm, "model_cost", {MODEL: {"supported_endpoints": ["/v1/responses"]}}
):
cfg = BedrockOpenAIResponsesConfig.for_model(f"chat_completions/{MODEL}")
assert isinstance(cfg, BedrockOpenAIResponsesConfig)
body = cfg.transform_responses_api_request(
model=f"chat_completions/{MODEL}",
input="hi",
response_api_optional_request_params={},
litellm_params=GenericLiteLLMParams(),
headers={},
)
assert body["model"] == MODEL
def test_converse_route_keeps_the_chat_completions_bridge(self):
with patch.object( # test-quality-ok: the gate reads the global cost map by design; no injection point exists
litellm, "model_cost", {MODEL: {"supported_endpoints": ["/v1/responses"]}}
):
assert BedrockOpenAIResponsesConfig.for_model(f"converse/{MODEL}") is None
class TestProviderResolution:
"""model_cost is patched explicitly: it is populated at import time from a GitHub

14
tests/unit/llms/bedrock/test_bedrock_common_utils.py
View file

@ -981,3 +981,17 @@ def test_unmapped_openai_family_model_routes_to_converse():
assert BedrockModelInfo.get_bedrock_route(unmapped) == "converse"
imported: Final = "bedrock/openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123"
assert BedrockModelInfo.get_bedrock_route(imported) == "openai"
@pytest.mark.parametrize(
("model", "expected"),
[
("converse/us.anthropic.claude-haiku-4-5-20251001-v1:0", "us.anthropic.claude-haiku-4-5-20251001-v1:0"),
("chat_completions/us.xai.grok-4.6", "us.xai.grok-4.6"),
("global.openai.gpt-5.6-sol", "global.openai.gpt-5.6-sol"),
],
)
def test_without_bedrock_route_prefix_hands_converse_the_bare_model_id(model, expected):
from litellm.llms.bedrock.common_utils import without_bedrock_route_prefix
assert without_bedrock_route_prefix(model) == expected

7
tests/unit/llms/bedrock/test_cross_region_inference_profile_mapping.py
View file

@ -138,9 +138,10 @@ def _bedrock_response(model, usage):
@pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id)
def test_bedrock_gpt_5_6_profiles_route_to_converse(profile, local_model_cost_map):
"""GPT-5.6 is served by Converse on bedrock-runtime, never by Invoke."""
assert BedrockModelInfo.get_bedrock_route(f"bedrock/{profile.model_id}") == "converse"
def test_bedrock_gpt_5_6_profiles_route_to_runtime_chat_completions(profile, local_model_cost_map):
"""GPT-5.6 is served by bedrock-runtime's native Chat Completions by default and by Converse when pinned, never by Invoke."""
assert BedrockModelInfo.get_bedrock_route(f"bedrock/{profile.model_id}") == "chat_completions"
assert BedrockModelInfo.get_bedrock_route(f"bedrock/converse/{profile.model_id}") == "converse"
@pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id)

2
tests/unit/test_utils.py
View file

@ -954,6 +954,8 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"supports_video_input": {"type": "boolean"},
"supports_vision": {"type": "boolean"},
"supports_web_search": {"type": "boolean"},
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": {"type": "boolean"},
"supports_bedrock_runtime_chat_completions_response_format": {"type": "boolean"},
"supports_url_context": {"type": "boolean"},
"supports_multimodal": {"type": "boolean"},
"uses_embed_content": {"type": "boolean"},

1
tests/unit/types/test_completion.py
View file

@ -181,6 +181,7 @@ def _build_dispatch_context() -> _CompletionDispatchContext:
optional_params={},
organization=None,
provider_config=None,
request_params={},
shared_session=None,
stream=None,
temperature=None,