scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-11 22:51:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 123

feat(bedrock): serve the OpenAI models on bedrock-runtime's native Responses API

Browse source 
AWS serves the OpenAI models on bedrock-runtime through an OpenAI-compatible
surface at /openai/v1/responses, alongside Converse. LiteLLM had no Responses
config for the bedrock provider, so /v1/responses fell back to the Chat
Completions bridge and was translated into Converse. A realistic Codex session
does not survive that translation: its function_call / function_call_output
history becomes Converse toolUse / toolResult blocks with no toolConfig, and
Converse rejects the request outright.

Add a Responses config for that surface, opted into per model from the price-map
supported_endpoints so models without the signal keep the bridge exactly as
before. Auth is Bearer when a Bedrock API key is present, SigV4 otherwise.

Both Bedrock endpoints reject the Codex history item types agent_message,
context_compaction and local_shell_call, so the normalization bedrock_mantle
carried privately moves into a shared module and both providers use it. They are
history items, so they only bite from the second turn onward -- a first-turn
smoke test passes and hides the problem. Verified against bedrock-runtime with
global.openai.gpt-5.6-sol: additional_tools is accepted there (unlike on
bedrock-mantle) while those three types are rejected, so the two endpoints do
not share one validator and each provider opts in explicitly.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Leonardo Freitas dos Santos 2026-08-27 10:31:31 +02:00
parent cd63c7e5a7
commit b849db94d2
No known key found for this signature in database
12 changed files with 695 additions and 132 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
litellm/__init__.py
View file

@ -1804,6 +1804,9 @@ if TYPE_CHECKING:
from .llms.openrouter.responses.transformation import (
OpenRouterResponsesAPIConfig as OpenRouterResponsesAPIConfig,
)
from .llms.bedrock.responses.transformation import (
BedrockOpenAIResponsesConfig as BedrockOpenAIResponsesConfig,
)
from .llms.bedrock_mantle.responses.transformation import (
BedrockMantleResponsesAPIConfig as BedrockMantleResponsesAPIConfig,
)

5
litellm/_lazy_imports_registry.py
View file

@ -241,6 +241,7 @@ LLM_CONFIG_NAMES: Final = (
"PerplexityResponsesConfig",
"DatabricksResponsesAPIConfig",
"OpenRouterResponsesAPIConfig",
"BedrockOpenAIResponsesConfig",
"BedrockMantleResponsesAPIConfig",
"GoogleAIStudioInteractionsConfig",
"VertexAIInteractionsConfig",
@ -897,6 +898,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"OpenAITextCompletionConfig",
),
"GroqChatConfig": (".llms.groq.chat.transformation", "GroqChatConfig"),
"BedrockOpenAIResponsesConfig": (
".llms.bedrock.responses.transformation",
"BedrockOpenAIResponsesConfig",
),
"BedrockMantleChatConfig": (
".llms.bedrock_mantle.chat.transformation",
"BedrockMantleChatConfig",

130
litellm/llms/base_llm/responses/codex_compat.py Normal file
View file

@ -0,0 +1,130 @@
"""Codex CLI wire-format quirks shared by the Responses API providers that need them.
Codex sends history item types that api.openai.com accepts but other Responses
backends reject with ``400 Invalid 'input': value did not match any expected
variant``. Both Amazon Bedrock endpoints reject them:
- ``bedrock-mantle.{region}.api.aws`` (verified against ``openai.gpt-5.6-sol``)
- ``bedrock-runtime.{region}.amazonaws.com/openai/v1`` (same, verified separately)
They are *history* items, so they only appear from the second turn of a session
onward -- a first-turn request succeeds and hides the problem entirely.
The normalizer is a pure transform that reports which types it rewrote; callers do
their own logging, so each provider keeps its own wording.
"""
import json
from collections.abc import Mapping
from typing import Final
from typing_extensions import ReadOnly, TypedDict
from litellm.types.llms.openai import ResponseInputParam
AGENT_MESSAGE_INPUT_ITEM_TYPE: Final = "agent_message"
CONTEXT_COMPACTION_INPUT_ITEM_TYPE: Final = "context_compaction"
LOCAL_SHELL_CALL_INPUT_ITEM_TYPE: Final = "local_shell_call"
class _RewrittenOutputTextBlock(TypedDict):
type: ReadOnly[str]
text: ReadOnly[str]
class _RewrittenAssistantMessageItem(TypedDict):
type: ReadOnly[str]
role: ReadOnly[str]
content: ReadOnly[tuple[_RewrittenOutputTextBlock, ...]]
class _RewrittenCompactionItem(TypedDict):
type: ReadOnly[str]
encrypted_content: ReadOnly[str]
class _RewrittenFunctionCallItem(TypedDict):
type: ReadOnly[str]
call_id: ReadOnly[str]
name: ReadOnly[str]
arguments: ReadOnly[str]
def _agent_message_text(item: "Mapping[str, object]") -> str:
content: Final = item.get("content")
if not isinstance(content, list):
return ""
return "".join(
str(block.get("text") or block.get("encrypted_content") or "") for block in content if isinstance(block, dict)
)
def _normalize_agent_message_item(item: "Mapping[str, object]") -> "_RewrittenAssistantMessageItem | None":
text: Final = _agent_message_text(item)
if not text:
return None
rewritten: Final[_RewrittenAssistantMessageItem] = {
"type": "message",
"role": "assistant",
"content": ({"type": "output_text", "text": text},),
}
return rewritten
def _normalize_context_compaction_item(item: "Mapping[str, object]") -> "_RewrittenCompactionItem | None":
encrypted_content: Final = item.get("encrypted_content")
if not isinstance(encrypted_content, str) or not encrypted_content:
return None
rewritten: Final[_RewrittenCompactionItem] = {"type": "compaction", "encrypted_content": encrypted_content}
return rewritten
def _normalize_local_shell_call_item(item: "Mapping[str, object]") -> "_RewrittenFunctionCallItem | None":
call_id: Final = item.get("call_id")
if not isinstance(call_id, str) or not call_id:
return None
action: Final = item.get("action")
rewritten: Final[_RewrittenFunctionCallItem] = {
"type": "function_call",
"call_id": call_id,
"name": "local_shell",
"arguments": json.dumps(action) if isinstance(action, dict) else "{}",
}
return rewritten
def _normalize_input_item(item: object) -> "tuple[object, str | None]":
"""Returns (normalized item, or None to drop it; original type when rewritten)."""
if not isinstance(item, dict):
return item, None
item_type: Final = item.get("type")
if item_type == AGENT_MESSAGE_INPUT_ITEM_TYPE:
return _normalize_agent_message_item(item), item_type
if item_type == CONTEXT_COMPACTION_INPUT_ITEM_TYPE:
return _normalize_context_compaction_item(item), item_type
if item_type == LOCAL_SHELL_CALL_INPUT_ITEM_TYPE:
return _normalize_local_shell_call_item(item), item_type
return item, None
def normalize_codex_input_items(
input: "str | ResponseInputParam",
) -> "tuple[str | ResponseInputParam, tuple[str, ...]]":
"""Rewrite the Codex history item types a Responses backend rejects.
``agent_message`` (Codex multi-agent traffic; its ``encrypted_content`` slot
carries the plaintext payload when the model never issued encrypted args)
becomes an assistant message, ``context_compaction`` becomes the ``compaction``
spelling these backends accept, and ``local_shell_call`` becomes the
``function_call`` its recorded ``function_call_output`` already pairs with.
Returns the normalized input and the sorted set of types that were rewritten,
so the caller can log in its own words. Non-list input is returned untouched.
"""
if not isinstance(input, list):
return input, ()
normalized: Final = tuple(_normalize_input_item(item) for item in input)
rewritten_types: Final = tuple(sorted(frozenset(item_type for _, item_type in normalized if item_type is not None)))
kept: Final = [i for i, _ in normalized if i is not None] # mutable-ok: downstream narrows on isinstance(list)
# Codex passthrough items sit outside the OpenAI input union.
return kept, rewritten_types # pyright: ignore[reportReturnType] # see above

22
litellm/llms/bedrock/common_utils.py
View file

@ -660,6 +660,28 @@ def strip_bedrock_throughput_suffix(model: str) -> str:
MANTLE_MESSAGES_PATH: Final = "/anthropic/v1/messages"
def bedrock_supports_openai_responses(model: str | None, model_cost: Mapping[str, object]) -> bool:
"""Whether a Bedrock model is served by bedrock-runtime's OpenAI Responses surface.
Purely data-driven from the model's price-map capability signal -- ``/v1/responses``
in ``supported_endpoints`` -- and overridable via ``register_model`` and proxy
``model_info``, so onboarding a model is a JSON change, never a code change.
There is deliberately no model-name match: AWS exposes this surface per model,
not per family, and the two Bedrock endpoints do not agree with each other
(bedrock-runtime accepts Codex's ``additional_tools`` items where
bedrock-mantle rejects them), so a name-shaped gate would be wrong.
A model absent from ``model_cost`` has no signal and returns False, leaving the
chat-completions bridge in place exactly as before.
"""
if not model:
return False
candidates: Final = (model_cost.get(key) for key in (model, f"bedrock/{model}"))
return any(
isinstance(entry, Mapping) and "/v1/responses" in (entry.get("supported_endpoints") or ())
for entry in candidates
)
def build_mantle_messages_url(
api_base: str | None,
aws_bedrock_runtime_endpoint: str | None,

143
litellm/llms/bedrock/responses/transformation.py Normal file
View file

@ -0,0 +1,143 @@
"""Amazon Bedrock Runtime - native OpenAI Responses API.
AWS serves the OpenAI models on ``bedrock-runtime`` through an OpenAI-compatible
surface at ``https://bedrock-runtime.{region}.amazonaws.com/openai/v1/responses``,
alongside Converse. Without this config the ``bedrock`` provider has no Responses
config at all, so ``/v1/responses`` falls back to the Chat Completions bridge and
the request is translated into Converse. A realistic Codex session does not
survive that translation: its ``function_call`` / ``function_call_output`` history
becomes Converse ``toolUse`` / ``toolResult`` blocks with no ``toolConfig``, and
Converse rejects the request outright with "The toolConfig field must be defined
when using toolUse and toolResult content blocks".
Payloads and SSE follow the OpenAI Responses spec, so this inherits
OpenAIResponsesAPIConfig and overrides only the endpoint URL, authentication, and
the Codex history-item normalization the endpoint requires.
Auth: Bearer token (litellm_params.api_key or the standard AWS_BEARER_TOKEN_BEDROCK)
when present; otherwise AWS SigV4 (service "bedrock") over the standard credential
chain, signed via BaseAWSLLM._sign_request once the body is final.
Model IDs: bedrock-runtime serves these models only through a cross-Region
inference profile, so the model is named ``us.openai.gpt-5.6-sol`` or
``global.openai.gpt-5.6-sol``; there is no in-Region form.
"""
from typing import Final
import litellm
from litellm._logging import verbose_logger
from litellm.llms.base_llm.responses.codex_compat import normalize_codex_input_items
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import bedrock_supports_openai_responses
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import ResponseInputParam
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
BEDROCK_RUNTIME_OPENAI_RESPONSES_PATH: Final = "/openai/v1/responses"
def resolve_bedrock_bearer_token(api_key: str | None) -> str | None:
return api_key or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
class BedrockOpenAIResponsesConfig(BaseAWSLLM, OpenAIResponsesAPIConfig):
"""Responses API config for the OpenAI models on the bedrock-runtime endpoint."""
@classmethod
def for_model(cls, model: str | None) -> "BedrockOpenAIResponsesConfig | None":
"""This config when ``model`` is served on the OpenAI Responses surface, else ``None``.
The capability decision lives here rather than in the shared dispatch so that
onboarding a model, or changing how the signal is read, stays inside the
Bedrock adapter. ``None`` leaves the caller's existing behaviour untouched --
chat-only Bedrock models keep the Chat Completions bridge.
"""
if not bedrock_supports_openai_responses(model, litellm.model_cost):
return None
return cls()
@property
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.BEDROCK
def get_complete_url(
self,
api_base: str | None,
litellm_params: dict, # mutable-ok: signature fixed by the BaseResponsesAPIConfig override contract
) -> str:
region: Final = self._get_aws_region_name(optional_params=litellm_params, model=None)
override: Final = (
api_base
or litellm_params.get("aws_bedrock_runtime_endpoint")
or get_secret_str("AWS_BEDROCK_RUNTIME_ENDPOINT")
)
host: Final = (override or f"https://bedrock-runtime.{region}.amazonaws.com").rstrip("/")
if host.endswith(BEDROCK_RUNTIME_OPENAI_RESPONSES_PATH):
return host
base: Final = next(
(host[: -len(suffix)] for suffix in ("/openai/v1", "/v1") if host.endswith(suffix)),
host,
)
return f"{base}{BEDROCK_RUNTIME_OPENAI_RESPONSES_PATH}"
def validate_environment(
self,
headers: dict, # mutable-ok: signature fixed by the BaseResponsesAPIConfig override contract
model: str,
litellm_params: GenericLiteLLMParams | None,
) -> dict: # mutable-ok: signature fixed by the BaseResponsesAPIConfig override contract
api_key: Final = litellm_params.api_key if litellm_params is not None else None
bearer: Final = resolve_bedrock_bearer_token(api_key)
if not bearer:
return headers
return {**headers, "Authorization": f"Bearer {bearer}"} # mutable-ok: dict return per the contract
def sign_request(
self,
headers: dict, # mutable-ok: signature fixed by the BaseResponsesAPIConfig override contract
optional_params: dict, # mutable-ok: same
request_data: dict, # mutable-ok: same
api_base: str,
api_key: str | None = None,
model: str | None = None,
stream: bool | None = None,
fake_stream: bool | None = None,
) -> "tuple[dict, bytes | None]": # mutable-ok: signature fixed by the override contract
if resolve_bedrock_bearer_token(api_key):
# Bedrock API keys are Bearer credentials; SigV4 on top would be wrong.
return headers, None
return self._sign_request(
service_name="bedrock",
headers=headers,
optional_params=optional_params,
request_data=request_data,
api_base=api_base,
model=model,
stream=stream,
fake_stream=fake_stream,
)
def transform_responses_api_request(
self,
model: str,
input: "str | ResponseInputParam",
response_api_optional_request_params: dict, # mutable-ok: signature fixed by the override contract
litellm_params: GenericLiteLLMParams,
headers: dict, # mutable-ok: same
) -> dict: # mutable-ok: same
normalized_input, rewritten_types = normalize_codex_input_items(input)
if rewritten_types:
verbose_logger.warning(
"Bedrock Runtime Responses API: rewrote Codex input item type(s) %s that the endpoint rejects.",
rewritten_types,
)
return super().transform_responses_api_request(
model=model,
input=normalized_input,
response_api_optional_request_params=response_api_optional_request_params,
litellm_params=litellm_params,
headers=headers,
)

124
litellm/llms/bedrock_mantle/responses/transformation.py
View file

@ -15,14 +15,11 @@ role / access key / profile / web identity), signed via the shared
BaseAWSLLM._sign_request after the request body is finalized.
"""
import json
from collections.abc import Mapping
from typing import Any, Final
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.llms.base_llm.responses.codex_compat import normalize_codex_input_items
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock_mantle.common_utils import (
MANTLE_HOST_RE,
@ -54,33 +51,6 @@ _BEDROCK_MANTLE_SUPPORTED_SERVICE_TIERS: Final = frozenset({"auto", "default"})
_CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools"
_CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE: Final = "agent_message"
_CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE: Final = "context_compaction"
_CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE: Final = "local_shell_call"
class _RewrittenOutputTextBlock(TypedDict):
type: ReadOnly[str]
text: ReadOnly[str]
class _RewrittenAssistantMessageItem(TypedDict):
type: ReadOnly[str]
role: ReadOnly[str]
content: ReadOnly[tuple[_RewrittenOutputTextBlock, ...]]
class _RewrittenCompactionItem(TypedDict):
type: ReadOnly[str]
encrypted_content: ReadOnly[str]
class _RewrittenFunctionCallItem(TypedDict):
type: ReadOnly[str]
call_id: ReadOnly[str]
name: ReadOnly[str]
arguments: ReadOnly[str]
class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPIConfig):
def __init__(
@ -186,7 +156,12 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
headers: dict,
) -> dict:
remaining_input, hoisted_tools = self._hoist_codex_additional_tools(input)
normalized_input: Final = self._normalize_codex_input_items(remaining_input)
normalized_input, rewritten_types = normalize_codex_input_items(remaining_input)
if rewritten_types:
verbose_logger.warning(
"Bedrock Mantle Responses API: rewrote Codex input item type(s) %s that Mantle rejects.",
list(rewritten_types),
)
request_params: Final = (
{
**response_api_optional_request_params,
@ -242,91 +217,6 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
)
return remaining_input, cls._filter_unsupported_tools(hoisted_tools)
@staticmethod
def _agent_message_text(item: "Mapping[str, object]") -> str:
content: Final = item.get("content")
if not isinstance(content, list):
return ""
return "".join(
str(block.get("text") or block.get("encrypted_content") or "")
for block in content
if isinstance(block, dict)
)
@classmethod
def _normalize_agent_message_item(cls, item: "Mapping[str, object]") -> "_RewrittenAssistantMessageItem | None":
text: Final = cls._agent_message_text(item)
if not text:
return None
rewritten: Final[_RewrittenAssistantMessageItem] = {
"type": "message",
"role": "assistant",
"content": ({"type": "output_text", "text": text},),
}
return rewritten
@staticmethod
def _normalize_context_compaction_item(item: "Mapping[str, object]") -> "_RewrittenCompactionItem | None":
encrypted_content: Final = item.get("encrypted_content")
if not isinstance(encrypted_content, str) or not encrypted_content:
return None
rewritten: Final[_RewrittenCompactionItem] = {"type": "compaction", "encrypted_content": encrypted_content}
return rewritten
@staticmethod
def _normalize_local_shell_call_item(item: "Mapping[str, object]") -> "_RewrittenFunctionCallItem | None":
call_id: Final = item.get("call_id")
if not isinstance(call_id, str) or not call_id:
return None
action: Final = item.get("action")
rewritten: Final[_RewrittenFunctionCallItem] = {
"type": "function_call",
"call_id": call_id,
"name": "local_shell",
"arguments": json.dumps(action) if isinstance(action, dict) else "{}",
}
return rewritten
@classmethod
def _normalize_codex_input_item(cls, item: object) -> "tuple[object, str | None]":
"""Returns (normalized item or None to drop it, original type when rewritten)."""
if not isinstance(item, dict):
return item, None
item_type: Final = item.get("type")
if item_type == _CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE:
return cls._normalize_agent_message_item(item), item_type
if item_type == _CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE:
return cls._normalize_context_compaction_item(item), item_type
if item_type == _CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE:
return cls._normalize_local_shell_call_item(item), item_type
return item, None
@classmethod
def _normalize_codex_input_items(
cls,
input: "str | ResponseInputParam",
) -> "str | ResponseInputParam":
"""Rewrite Codex history item types Mantle rejects with 400 "Invalid
'input': value did not match any expected variant" into supported
equivalents. `agent_message` (Codex multi-agent traffic; its
encrypted_content slot carries the plaintext payload when the model
never issued encrypted args) becomes an assistant message,
`context_compaction` becomes the `compaction` spelling Mantle accepts,
and `local_shell_call` becomes the function_call its recorded
function_call_output already pairs with.
"""
if not isinstance(input, list):
return input
normalized: Final = tuple(cls._normalize_codex_input_item(item) for item in input)
rewritten_types: Final = sorted(frozenset(item_type for _, item_type in normalized if item_type is not None))
if rewritten_types:
verbose_logger.warning(
"Bedrock Mantle Responses API: rewrote Codex input item type(s) %s that Mantle rejects.",
rewritten_types,
)
kept: Final = [item for item, _ in normalized if item is not None] # mutable-ok: ResponseInputParam is a list
return kept # pyright: ignore[reportReturnType] # Codex passthrough items sit outside the OpenAI input union
def map_openai_params(
self,
response_api_optional_params: ResponsesAPIOptionalRequestParams,

30
litellm/model_prices_and_context_window_backup.json
View file

@ -49672,7 +49672,10 @@
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
"supports_vision": true,
"supported_endpoints": [
"/v1/responses"
]
},
"global.openai.gpt-5.6-sol": {
"input_cost_per_token": 5e-06,
@ -49698,7 +49701,10 @@
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
"supports_vision": true,
"supported_endpoints": [
"/v1/responses"
]
},
"us.openai.gpt-5.6-terra": {
"input_cost_per_token": 2.2e-06,
@ -49724,7 +49730,10 @@
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
"supports_vision": true,
"supported_endpoints": [
"/v1/responses"
]
},
"global.openai.gpt-5.6-terra": {
"input_cost_per_token": 2e-06,
@ -49750,7 +49759,10 @@
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
"supports_vision": true,
"supported_endpoints": [
"/v1/responses"
]
},
"us.openai.gpt-5.6-luna": {
"input_cost_per_token": 2.2e-07,
@ -49776,7 +49788,10 @@
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
"supports_vision": true,
"supported_endpoints": [
"/v1/responses"
]
},
"global.openai.gpt-5.6-luna": {
"input_cost_per_token": 2e-07,
@ -49802,7 +49817,10 @@
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
"supports_vision": true,
"supported_endpoints": [
"/v1/responses"
]
},
"bedrock_mantle/openai.gpt-5.5": {
"input_cost_per_token": 5.5e-06,

5
litellm/utils.py
View file

@ -8593,6 +8593,11 @@ class ProviderConfigManager:
return litellm.OpenRouterResponsesAPIConfig()
elif litellm.LlmProviders.HOSTED_VLLM == provider:
return litellm.HostedVLLMResponsesAPIConfig()
elif litellm.LlmProviders.BEDROCK == provider:
# bedrock-runtime serves the OpenAI models on an OpenAI-compatible surface
# (/openai/v1/responses) alongside Converse. The adapter decides whether a
# given model is on it; None keeps the chat-completions bridge.
return litellm.BedrockOpenAIResponsesConfig.for_model(model)
elif litellm.LlmProviders.BEDROCK_MANTLE == provider:
# Both decisions are data-driven from the model's price-map entry, with
# no model-name logic. Capability (can it serve Responses?) comes from

30
model_prices_and_context_window.json
View file

@ -49672,7 +49672,10 @@
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
"supports_vision": true,
"supported_endpoints": [
"/v1/responses"
]
},
"global.openai.gpt-5.6-sol": {
"input_cost_per_token": 5e-06,
@ -49698,7 +49701,10 @@
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
"supports_vision": true,
"supported_endpoints": [
"/v1/responses"
]
},
"us.openai.gpt-5.6-terra": {
"input_cost_per_token": 2.2e-06,
@ -49724,7 +49730,10 @@
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
"supports_vision": true,
"supported_endpoints": [
"/v1/responses"
]
},
"global.openai.gpt-5.6-terra": {
"input_cost_per_token": 2e-06,
@ -49750,7 +49759,10 @@
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
"supports_vision": true,
"supported_endpoints": [
"/v1/responses"
]
},
"us.openai.gpt-5.6-luna": {
"input_cost_per_token": 2.2e-07,
@ -49776,7 +49788,10 @@
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
"supports_vision": true,
"supported_endpoints": [
"/v1/responses"
]
},
"global.openai.gpt-5.6-luna": {
"input_cost_per_token": 2e-07,
@ -49802,7 +49817,10 @@
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
"supports_vision": true,
"supported_endpoints": [
"/v1/responses"
]
},
"bedrock_mantle/openai.gpt-5.5": {
"input_cost_per_token": 5.5e-06,

116
tests/test_litellm/llms/base_llm/responses/test_codex_compat.py Normal file
View file

@ -0,0 +1,116 @@
"""Shared Codex wire-format normalization.
Both Bedrock endpoints reject the Codex *history* item types with
``400 Invalid 'input': value did not match any expected variant``. They are history
items, so they only appear from the second turn of a session onward — a first-turn
smoke test passes and hides the problem entirely.
"""
import json
import pytest
from litellm.llms.base_llm.responses.codex_compat import normalize_codex_input_items
USER = {"role": "user", "content": "hi"}
class TestAgentMessage:
def test_becomes_an_assistant_message(self):
item = {
"type": "agent_message",
"role": "assistant",
"content": [{"type": "output_text", "text": "prior turn"}],
}
out, types = normalize_codex_input_items([item, USER])
assert types == ("agent_message",)
assert out[0] == {
"type": "message",
"role": "assistant",
"content": ({"type": "output_text", "text": "prior turn"},),
}
def test_encrypted_content_slot_is_used_as_text(self):
"""Codex puts the plaintext payload there when the model issued no encrypted args."""
item = {"type": "agent_message", "content": [{"encrypted_content": "plain"}]}
out, _ = normalize_codex_input_items([item, USER])
assert out[0]["content"] == ({"type": "output_text", "text": "plain"},)
def test_non_list_content_yields_no_text_and_drops_the_item(self):
out, types = normalize_codex_input_items([{"type": "agent_message", "content": "not a list"}, USER])
assert out == [USER]
assert types == ("agent_message",)
def test_textless_item_is_dropped(self):
out, types = normalize_codex_input_items([{"type": "agent_message", "content": []}, USER])
assert out == [USER]
assert types == ("agent_message",)
class TestContextCompaction:
def test_becomes_compaction(self):
out, types = normalize_codex_input_items([{"type": "context_compaction", "encrypted_content": "abc"}, USER])
assert out[0] == {"type": "compaction", "encrypted_content": "abc"}
assert types == ("context_compaction",)
@pytest.mark.parametrize("bad", [{}, {"encrypted_content": ""}, {"encrypted_content": 7}])
def test_without_usable_content_is_dropped(self, bad):
out, _ = normalize_codex_input_items([{"type": "context_compaction", **bad}, USER])
assert out == [USER]
class TestLocalShellCall:
def test_becomes_the_function_call_its_output_pairs_with(self):
out, types = normalize_codex_input_items(
[{"type": "local_shell_call", "call_id": "c1", "action": {"command": ["ls"]}}, USER]
)
assert out[0] == {
"type": "function_call",
"call_id": "c1",
"name": "local_shell",
"arguments": json.dumps({"command": ["ls"]}),
}
assert types == ("local_shell_call",)
def test_missing_action_yields_empty_arguments(self):
out, _ = normalize_codex_input_items([{"type": "local_shell_call", "call_id": "c1"}, USER])
assert out[0]["arguments"] == "{}"
def test_without_call_id_is_dropped(self):
out, _ = normalize_codex_input_items([{"type": "local_shell_call"}, USER])
assert out == [USER]
class TestPassthroughAndShape:
def test_string_input_untouched(self):
assert normalize_codex_input_items("just a prompt") == ("just a prompt", ())
def test_unrelated_items_untouched_and_no_types_reported(self):
items = [USER, {"type": "message", "role": "assistant", "content": []}]
out, types = normalize_codex_input_items(items)
assert out == items
assert types == ()
def test_non_mapping_entries_pass_through_except_a_literal_none(self):
"""A literal ``None`` is indistinguishable from "drop this item" in the
per-item return protocol, so it is dropped. Other non-mapping entries pass
through untouched. This matches the behaviour before the normalizer moved
out of the bedrock_mantle config."""
out, types = normalize_codex_input_items(["a string", 42, None, USER])
assert out == ["a string", 42, USER]
assert types == ()
def test_types_are_sorted_and_deduplicated(self):
items = [
{"type": "local_shell_call", "call_id": "c1"},
{"type": "agent_message", "content": [{"text": "x"}]},
{"type": "local_shell_call", "call_id": "c2"},
]
_, types = normalize_codex_input_items(items)
assert types == ("agent_message", "local_shell_call")
def test_returns_a_list_not_a_tuple(self):
"""The input->messages conversion downstream narrows on isinstance(input, list);
a tuple silently yields zero messages and the provider rejects the request."""
out, _ = normalize_codex_input_items([{"type": "agent_message", "content": [{"text": "x"}]}, USER])
assert isinstance(out, list)

211
tests/test_litellm/llms/bedrock/responses/test_bedrock_openai_responses.py Normal file
View file

@ -0,0 +1,211 @@
"""Native OpenAI Responses API on the bedrock-runtime endpoint.
Without this config the bedrock provider has no Responses config, so /v1/responses
falls back to the Chat Completions bridge and rides Converse.
"""
import json
from importlib.resources import files
from unittest.mock import patch
import pytest
import litellm
from litellm.llms.bedrock.common_utils import bedrock_supports_openai_responses
from litellm.llms.bedrock.responses.transformation import BedrockOpenAIResponsesConfig
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
MODEL = "global.openai.gpt-5.6-sol"
def _cfg():
return BedrockOpenAIResponsesConfig()
class TestCompleteURL:
def test_default_host_and_path(self):
url = _cfg().get_complete_url(None, {"aws_region_name": "us-east-1"})
assert url == "https://bedrock-runtime.us-east-1.amazonaws.com/openai/v1/responses"
def test_region_is_honoured(self):
url = _cfg().get_complete_url(None, {"aws_region_name": "eu-west-1"})
assert url == "https://bedrock-runtime.eu-west-1.amazonaws.com/openai/v1/responses"
@pytest.mark.parametrize(
"api_base",
[
"https://proxy.example.com",
"https://proxy.example.com/",
"https://proxy.example.com/openai/v1",
"https://proxy.example.com/openai/v1/responses",
],
)
def test_custom_host_is_preserved_and_path_never_doubles(self, api_base):
url = _cfg().get_complete_url(api_base, {"aws_region_name": "us-east-1"})
assert url == "https://proxy.example.com/openai/v1/responses"
def test_runtime_endpoint_param_is_honoured(self):
url = _cfg().get_complete_url(
None, {"aws_region_name": "us-east-1", "aws_bedrock_runtime_endpoint": "https://vpce.example.com"}
)
assert url == "https://vpce.example.com/openai/v1/responses"
class TestAuth:
def test_bearer_token_is_used_when_present(self):
headers = _cfg().validate_environment({}, MODEL, GenericLiteLLMParams(api_key="sk-bedrock"))
assert headers["Authorization"] == "Bearer sk-bedrock"
def test_no_authorization_header_without_a_token(self, monkeypatch):
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
headers = _cfg().validate_environment({}, MODEL, GenericLiteLLMParams())
assert "Authorization" not in headers
def test_sigv4_is_skipped_when_a_bearer_token_is_present(self):
"""Bedrock API keys are Bearer; signing on top would be wrong."""
headers, body = _cfg().sign_request(
headers={"Authorization": "Bearer sk-bedrock"},
optional_params={},
request_data={},
api_base="https://bedrock-runtime.us-east-1.amazonaws.com/openai/v1/responses",
api_key="sk-bedrock",
)
assert headers["Authorization"] == "Bearer sk-bedrock"
assert body is None
class TestProviderIdentity:
def test_reports_the_bedrock_provider(self):
"""Cost tracking and callbacks key off this, so it must stay `bedrock` rather
than becoming a separate provider."""
assert _cfg().custom_llm_provider == LlmProviders.BEDROCK
class TestSigV4Fallback:
def test_signs_with_sigv4_when_no_bearer_token_is_present(self, monkeypatch):
"""No Bedrock API key means SigV4 over the standard credential chain. Static
credentials are set in the environment so signing stays a local computation."""
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
headers, body = _cfg().sign_request(
headers={"content-type": "application/json"},
optional_params={"aws_region_name": "us-east-1"},
request_data={"model": MODEL, "input": "hi"},
api_base="https://bedrock-runtime.us-east-1.amazonaws.com/openai/v1/responses",
api_key=None,
)
assert "Authorization" in headers
assert headers["Authorization"].startswith("AWS4-HMAC-SHA256")
assert "Credential=AKIAIOSFODNN7EXAMPLE" in headers["Authorization"]
class TestPriceMapGate:
def test_absent_model_has_no_signal(self):
assert bedrock_supports_openai_responses(MODEL, {}) is False
def test_none_model_is_false(self):
assert bedrock_supports_openai_responses(None, {}) is False
def test_signal_on_the_bare_key(self):
cost = {MODEL: {"supported_endpoints": ["/v1/responses"]}}
assert bedrock_supports_openai_responses(MODEL, cost) is True
def test_signal_on_the_bedrock_prefixed_key(self):
cost = {f"bedrock/{MODEL}": {"supported_endpoints": ["/v1/responses"]}}
assert bedrock_supports_openai_responses(MODEL, cost) is True
def test_other_endpoints_do_not_count(self):
cost = {MODEL: {"supported_endpoints": ["/v1/messages"]}}
assert bedrock_supports_openai_responses(MODEL, cost) is False
class TestForModelGate:
"""The capability decision lives on the adapter, not in the shared dispatch."""
def test_returns_a_config_for_a_signalled_model(self):
with patch.object( # test-quality-ok: the gate reads the global cost map by design; no injection point exists
litellm, "model_cost", {MODEL: {"supported_endpoints": ["/v1/responses"]}}
):
assert isinstance(BedrockOpenAIResponsesConfig.for_model(MODEL), BedrockOpenAIResponsesConfig)
def test_returns_none_for_an_unsignalled_model(self):
with patch.object( # test-quality-ok: the gate reads the global cost map by design; no injection point exists
litellm, "model_cost", {}
):
assert BedrockOpenAIResponsesConfig.for_model(MODEL) is None
def test_returns_none_for_no_model(self):
with patch.object( # test-quality-ok: the gate reads the global cost map by design; no injection point exists
litellm, "model_cost", {}
):
assert BedrockOpenAIResponsesConfig.for_model(None) is None
class TestProviderResolution:
"""model_cost is patched explicitly: it is populated at import time from a GitHub
fetch unless LITELLM_LOCAL_MODEL_COST_MAP is set, and conftest's monkeypatch of
that variable lands after import — so these must not read the global."""
def test_signalled_model_resolves_to_the_bedrock_responses_config(self):
with patch.object( # test-quality-ok: resolution reads the global cost map by design; no HTTP boundary or injection point exists
litellm, "model_cost", {MODEL: {"supported_endpoints": ["/v1/responses"]}}
):
cfg = ProviderConfigManager.get_provider_responses_api_config(model=MODEL, provider=LlmProviders.BEDROCK)
assert isinstance(cfg, BedrockOpenAIResponsesConfig)
def test_unsignalled_model_keeps_the_existing_bridge(self):
"""Claude on Bedrock has no OpenAI surface; it must keep falling through to
the chat-completions bridge exactly as before."""
with patch.object(litellm, "model_cost", {}): # test-quality-ok: resolution reads the global cost map by design
cfg = ProviderConfigManager.get_provider_responses_api_config(
model="anthropic.claude-3-haiku-20240307-v1:0", provider=LlmProviders.BEDROCK
)
assert cfg is None
def test_the_shipped_price_map_signals_the_gpt_56_family(self):
"""Reads the bundled backup directly rather than the network-fetched global."""
shipped = json.loads(
files("litellm").joinpath("model_prices_and_context_window_backup.json").read_text(encoding="utf-8")
)
for prefix in ("us", "global"):
for variant in ("sol", "terra", "luna"):
model = f"{prefix}.openai.gpt-5.6-{variant}"
assert bedrock_supports_openai_responses(model, shipped) is True, model
class TestCodexHistoryNormalization:
def test_history_items_the_endpoint_rejects_are_rewritten(self):
body = _cfg().transform_responses_api_request(
model=MODEL,
input=[
{"type": "agent_message", "content": [{"type": "output_text", "text": "prior"}]},
{"type": "context_compaction", "encrypted_content": "abc"},
{"type": "local_shell_call", "call_id": "c1", "action": {"command": ["ls"]}},
{"role": "user", "content": "carry on"},
],
response_api_optional_request_params={},
litellm_params=GenericLiteLLMParams(),
headers={},
)
assert [i.get("type") or i.get("role") for i in body["input"]] == [
"message",
"compaction",
"function_call",
"user",
]
def test_a_first_turn_request_is_untouched(self):
"""The rejected types are history items, so turn one exercises none of this."""
original = [{"role": "user", "content": "first turn"}]
body = _cfg().transform_responses_api_request(
model=MODEL,
input=list(original),
response_api_optional_request_params={},
litellm_params=GenericLiteLLMParams(),
headers={},
)
assert body["input"] == original

8
tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py
View file

@ -287,9 +287,11 @@ def test_bedrock_gpt_5_6_advertises_only_converse_supported_features(
raw = _packaged_cost_map()[profile.model_id]
assert raw["supported_modalities"] == ["text", "image"]
assert raw["supported_output_modalities"] == ["text"]
# No bedrock_converse entry declares supported_endpoints; these models are reachable
# on chat completions and on the Responses API without it.
assert "supported_endpoints" not in raw
# supported_endpoints opts these models into bedrock-runtime's native OpenAI
# Responses surface (/openai/v1/responses), which AWS serves alongside Converse.
# It is the only signal that selects it; without the entry they fall back to the
# Chat Completions bridge, as every other bedrock_converse model still does.
assert raw["supported_endpoints"] == ["/v1/responses"]
@pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id)