From b849db94d23ec2825fc6d858e37b874d84e1e22d Mon Sep 17 00:00:00 2001 From: Leonardo Freitas dos Santos Date: Thu, 27 Aug 2026 10:31:31 +0200 Subject: [PATCH] feat(bedrock): serve the OpenAI models on bedrock-runtime's native Responses API AWS serves the OpenAI models on bedrock-runtime through an OpenAI-compatible surface at /openai/v1/responses, alongside Converse. LiteLLM had no Responses config for the bedrock provider, so /v1/responses fell back to the Chat Completions bridge and was translated into Converse. A realistic Codex session does not survive that translation: its function_call / function_call_output history becomes Converse toolUse / toolResult blocks with no toolConfig, and Converse rejects the request outright. Add a Responses config for that surface, opted into per model from the price-map supported_endpoints so models without the signal keep the bridge exactly as before. Auth is Bearer when a Bedrock API key is present, SigV4 otherwise. Both Bedrock endpoints reject the Codex history item types agent_message, context_compaction and local_shell_call, so the normalization bedrock_mantle carried privately moves into a shared module and both providers use it. They are history items, so they only bite from the second turn onward -- a first-turn smoke test passes and hides the problem. Verified against bedrock-runtime with global.openai.gpt-5.6-sol: additional_tools is accepted there (unlike on bedrock-mantle) while those three types are rejected, so the two endpoints do not share one validator and each provider opts in explicitly. Co-Authored-By: Claude Opus 5 (1M context) --- litellm/__init__.py | 3 + litellm/_lazy_imports_registry.py | 5 + .../llms/base_llm/responses/codex_compat.py | 130 +++++++++++ litellm/llms/bedrock/common_utils.py | 22 ++ .../llms/bedrock/responses/transformation.py | 143 ++++++++++++ .../responses/transformation.py | 124 +--------- ...odel_prices_and_context_window_backup.json | 30 ++- litellm/utils.py | 5 + model_prices_and_context_window.json | 30 ++- .../base_llm/responses/test_codex_compat.py | 116 ++++++++++ .../test_bedrock_openai_responses.py | 211 ++++++++++++++++++ ..._cross_region_inference_profile_mapping.py | 8 +- 12 files changed, 695 insertions(+), 132 deletions(-) create mode 100644 litellm/llms/base_llm/responses/codex_compat.py create mode 100644 litellm/llms/bedrock/responses/transformation.py create mode 100644 tests/test_litellm/llms/base_llm/responses/test_codex_compat.py create mode 100644 tests/test_litellm/llms/bedrock/responses/test_bedrock_openai_responses.py diff --git a/litellm/__init__.py b/litellm/__init__.py index eebd2dad91e..da293d32b2d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1804,6 +1804,9 @@ if TYPE_CHECKING: from .llms.openrouter.responses.transformation import ( OpenRouterResponsesAPIConfig as OpenRouterResponsesAPIConfig, ) + from .llms.bedrock.responses.transformation import ( + BedrockOpenAIResponsesConfig as BedrockOpenAIResponsesConfig, + ) from .llms.bedrock_mantle.responses.transformation import ( BedrockMantleResponsesAPIConfig as BedrockMantleResponsesAPIConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 1c833256598..15389791a12 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -241,6 +241,7 @@ LLM_CONFIG_NAMES: Final = ( "PerplexityResponsesConfig", "DatabricksResponsesAPIConfig", "OpenRouterResponsesAPIConfig", + "BedrockOpenAIResponsesConfig", "BedrockMantleResponsesAPIConfig", "GoogleAIStudioInteractionsConfig", "VertexAIInteractionsConfig", @@ -897,6 +898,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "OpenAITextCompletionConfig", ), "GroqChatConfig": (".llms.groq.chat.transformation", "GroqChatConfig"), + "BedrockOpenAIResponsesConfig": ( + ".llms.bedrock.responses.transformation", + "BedrockOpenAIResponsesConfig", + ), "BedrockMantleChatConfig": ( ".llms.bedrock_mantle.chat.transformation", "BedrockMantleChatConfig", diff --git a/litellm/llms/base_llm/responses/codex_compat.py b/litellm/llms/base_llm/responses/codex_compat.py new file mode 100644 index 00000000000..5faf0946e96 --- /dev/null +++ b/litellm/llms/base_llm/responses/codex_compat.py @@ -0,0 +1,130 @@ +"""Codex CLI wire-format quirks shared by the Responses API providers that need them. + +Codex sends history item types that api.openai.com accepts but other Responses +backends reject with ``400 Invalid 'input': value did not match any expected +variant``. Both Amazon Bedrock endpoints reject them: + +- ``bedrock-mantle.{region}.api.aws`` (verified against ``openai.gpt-5.6-sol``) +- ``bedrock-runtime.{region}.amazonaws.com/openai/v1`` (same, verified separately) + +They are *history* items, so they only appear from the second turn of a session +onward -- a first-turn request succeeds and hides the problem entirely. + +The normalizer is a pure transform that reports which types it rewrote; callers do +their own logging, so each provider keeps its own wording. +""" + +import json +from collections.abc import Mapping +from typing import Final + +from typing_extensions import ReadOnly, TypedDict + +from litellm.types.llms.openai import ResponseInputParam + +AGENT_MESSAGE_INPUT_ITEM_TYPE: Final = "agent_message" +CONTEXT_COMPACTION_INPUT_ITEM_TYPE: Final = "context_compaction" +LOCAL_SHELL_CALL_INPUT_ITEM_TYPE: Final = "local_shell_call" + + +class _RewrittenOutputTextBlock(TypedDict): + type: ReadOnly[str] + text: ReadOnly[str] + + +class _RewrittenAssistantMessageItem(TypedDict): + type: ReadOnly[str] + role: ReadOnly[str] + content: ReadOnly[tuple[_RewrittenOutputTextBlock, ...]] + + +class _RewrittenCompactionItem(TypedDict): + type: ReadOnly[str] + encrypted_content: ReadOnly[str] + + +class _RewrittenFunctionCallItem(TypedDict): + type: ReadOnly[str] + call_id: ReadOnly[str] + name: ReadOnly[str] + arguments: ReadOnly[str] + + +def _agent_message_text(item: "Mapping[str, object]") -> str: + content: Final = item.get("content") + if not isinstance(content, list): + return "" + return "".join( + str(block.get("text") or block.get("encrypted_content") or "") for block in content if isinstance(block, dict) + ) + + +def _normalize_agent_message_item(item: "Mapping[str, object]") -> "_RewrittenAssistantMessageItem | None": + text: Final = _agent_message_text(item) + if not text: + return None + rewritten: Final[_RewrittenAssistantMessageItem] = { + "type": "message", + "role": "assistant", + "content": ({"type": "output_text", "text": text},), + } + return rewritten + + +def _normalize_context_compaction_item(item: "Mapping[str, object]") -> "_RewrittenCompactionItem | None": + encrypted_content: Final = item.get("encrypted_content") + if not isinstance(encrypted_content, str) or not encrypted_content: + return None + rewritten: Final[_RewrittenCompactionItem] = {"type": "compaction", "encrypted_content": encrypted_content} + return rewritten + + +def _normalize_local_shell_call_item(item: "Mapping[str, object]") -> "_RewrittenFunctionCallItem | None": + call_id: Final = item.get("call_id") + if not isinstance(call_id, str) or not call_id: + return None + action: Final = item.get("action") + rewritten: Final[_RewrittenFunctionCallItem] = { + "type": "function_call", + "call_id": call_id, + "name": "local_shell", + "arguments": json.dumps(action) if isinstance(action, dict) else "{}", + } + return rewritten + + +def _normalize_input_item(item: object) -> "tuple[object, str | None]": + """Returns (normalized item, or None to drop it; original type when rewritten).""" + if not isinstance(item, dict): + return item, None + item_type: Final = item.get("type") + if item_type == AGENT_MESSAGE_INPUT_ITEM_TYPE: + return _normalize_agent_message_item(item), item_type + if item_type == CONTEXT_COMPACTION_INPUT_ITEM_TYPE: + return _normalize_context_compaction_item(item), item_type + if item_type == LOCAL_SHELL_CALL_INPUT_ITEM_TYPE: + return _normalize_local_shell_call_item(item), item_type + return item, None + + +def normalize_codex_input_items( + input: "str | ResponseInputParam", +) -> "tuple[str | ResponseInputParam, tuple[str, ...]]": + """Rewrite the Codex history item types a Responses backend rejects. + + ``agent_message`` (Codex multi-agent traffic; its ``encrypted_content`` slot + carries the plaintext payload when the model never issued encrypted args) + becomes an assistant message, ``context_compaction`` becomes the ``compaction`` + spelling these backends accept, and ``local_shell_call`` becomes the + ``function_call`` its recorded ``function_call_output`` already pairs with. + + Returns the normalized input and the sorted set of types that were rewritten, + so the caller can log in its own words. Non-list input is returned untouched. + """ + if not isinstance(input, list): + return input, () + normalized: Final = tuple(_normalize_input_item(item) for item in input) + rewritten_types: Final = tuple(sorted(frozenset(item_type for _, item_type in normalized if item_type is not None))) + kept: Final = [i for i, _ in normalized if i is not None] # mutable-ok: downstream narrows on isinstance(list) + # Codex passthrough items sit outside the OpenAI input union. + return kept, rewritten_types # pyright: ignore[reportReturnType] # see above diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 4ad20772ed0..b08d181bea1 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -660,6 +660,28 @@ def strip_bedrock_throughput_suffix(model: str) -> str: MANTLE_MESSAGES_PATH: Final = "/anthropic/v1/messages" +def bedrock_supports_openai_responses(model: str | None, model_cost: Mapping[str, object]) -> bool: + """Whether a Bedrock model is served by bedrock-runtime's OpenAI Responses surface. + + Purely data-driven from the model's price-map capability signal -- ``/v1/responses`` + in ``supported_endpoints`` -- and overridable via ``register_model`` and proxy + ``model_info``, so onboarding a model is a JSON change, never a code change. + There is deliberately no model-name match: AWS exposes this surface per model, + not per family, and the two Bedrock endpoints do not agree with each other + (bedrock-runtime accepts Codex's ``additional_tools`` items where + bedrock-mantle rejects them), so a name-shaped gate would be wrong. + A model absent from ``model_cost`` has no signal and returns False, leaving the + chat-completions bridge in place exactly as before. + """ + if not model: + return False + candidates: Final = (model_cost.get(key) for key in (model, f"bedrock/{model}")) + return any( + isinstance(entry, Mapping) and "/v1/responses" in (entry.get("supported_endpoints") or ()) + for entry in candidates + ) + + def build_mantle_messages_url( api_base: str | None, aws_bedrock_runtime_endpoint: str | None, diff --git a/litellm/llms/bedrock/responses/transformation.py b/litellm/llms/bedrock/responses/transformation.py new file mode 100644 index 00000000000..f973590b2c8 --- /dev/null +++ b/litellm/llms/bedrock/responses/transformation.py @@ -0,0 +1,143 @@ +"""Amazon Bedrock Runtime - native OpenAI Responses API. + +AWS serves the OpenAI models on ``bedrock-runtime`` through an OpenAI-compatible +surface at ``https://bedrock-runtime.{region}.amazonaws.com/openai/v1/responses``, +alongside Converse. Without this config the ``bedrock`` provider has no Responses +config at all, so ``/v1/responses`` falls back to the Chat Completions bridge and +the request is translated into Converse. A realistic Codex session does not +survive that translation: its ``function_call`` / ``function_call_output`` history +becomes Converse ``toolUse`` / ``toolResult`` blocks with no ``toolConfig``, and +Converse rejects the request outright with "The toolConfig field must be defined +when using toolUse and toolResult content blocks". + +Payloads and SSE follow the OpenAI Responses spec, so this inherits +OpenAIResponsesAPIConfig and overrides only the endpoint URL, authentication, and +the Codex history-item normalization the endpoint requires. + +Auth: Bearer token (litellm_params.api_key or the standard AWS_BEARER_TOKEN_BEDROCK) +when present; otherwise AWS SigV4 (service "bedrock") over the standard credential +chain, signed via BaseAWSLLM._sign_request once the body is final. + +Model IDs: bedrock-runtime serves these models only through a cross-Region +inference profile, so the model is named ``us.openai.gpt-5.6-sol`` or +``global.openai.gpt-5.6-sol``; there is no in-Region form. +""" + +from typing import Final + +import litellm +from litellm._logging import verbose_logger +from litellm.llms.base_llm.responses.codex_compat import normalize_codex_input_items +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.common_utils import bedrock_supports_openai_responses +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ResponseInputParam +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + +BEDROCK_RUNTIME_OPENAI_RESPONSES_PATH: Final = "/openai/v1/responses" + + +def resolve_bedrock_bearer_token(api_key: str | None) -> str | None: + return api_key or get_secret_str("AWS_BEARER_TOKEN_BEDROCK") + + +class BedrockOpenAIResponsesConfig(BaseAWSLLM, OpenAIResponsesAPIConfig): + """Responses API config for the OpenAI models on the bedrock-runtime endpoint.""" + + @classmethod + def for_model(cls, model: str | None) -> "BedrockOpenAIResponsesConfig | None": + """This config when ``model`` is served on the OpenAI Responses surface, else ``None``. + + The capability decision lives here rather than in the shared dispatch so that + onboarding a model, or changing how the signal is read, stays inside the + Bedrock adapter. ``None`` leaves the caller's existing behaviour untouched -- + chat-only Bedrock models keep the Chat Completions bridge. + """ + if not bedrock_supports_openai_responses(model, litellm.model_cost): + return None + return cls() + + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.BEDROCK + + def get_complete_url( + self, + api_base: str | None, + litellm_params: dict, # mutable-ok: signature fixed by the BaseResponsesAPIConfig override contract + ) -> str: + region: Final = self._get_aws_region_name(optional_params=litellm_params, model=None) + override: Final = ( + api_base + or litellm_params.get("aws_bedrock_runtime_endpoint") + or get_secret_str("AWS_BEDROCK_RUNTIME_ENDPOINT") + ) + host: Final = (override or f"https://bedrock-runtime.{region}.amazonaws.com").rstrip("/") + if host.endswith(BEDROCK_RUNTIME_OPENAI_RESPONSES_PATH): + return host + base: Final = next( + (host[: -len(suffix)] for suffix in ("/openai/v1", "/v1") if host.endswith(suffix)), + host, + ) + return f"{base}{BEDROCK_RUNTIME_OPENAI_RESPONSES_PATH}" + + def validate_environment( + self, + headers: dict, # mutable-ok: signature fixed by the BaseResponsesAPIConfig override contract + model: str, + litellm_params: GenericLiteLLMParams | None, + ) -> dict: # mutable-ok: signature fixed by the BaseResponsesAPIConfig override contract + api_key: Final = litellm_params.api_key if litellm_params is not None else None + bearer: Final = resolve_bedrock_bearer_token(api_key) + if not bearer: + return headers + return {**headers, "Authorization": f"Bearer {bearer}"} # mutable-ok: dict return per the contract + + def sign_request( + self, + headers: dict, # mutable-ok: signature fixed by the BaseResponsesAPIConfig override contract + optional_params: dict, # mutable-ok: same + request_data: dict, # mutable-ok: same + api_base: str, + api_key: str | None = None, + model: str | None = None, + stream: bool | None = None, + fake_stream: bool | None = None, + ) -> "tuple[dict, bytes | None]": # mutable-ok: signature fixed by the override contract + if resolve_bedrock_bearer_token(api_key): + # Bedrock API keys are Bearer credentials; SigV4 on top would be wrong. + return headers, None + return self._sign_request( + service_name="bedrock", + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + model=model, + stream=stream, + fake_stream=fake_stream, + ) + + def transform_responses_api_request( + self, + model: str, + input: "str | ResponseInputParam", + response_api_optional_request_params: dict, # mutable-ok: signature fixed by the override contract + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: same + ) -> dict: # mutable-ok: same + normalized_input, rewritten_types = normalize_codex_input_items(input) + if rewritten_types: + verbose_logger.warning( + "Bedrock Runtime Responses API: rewrote Codex input item type(s) %s that the endpoint rejects.", + rewritten_types, + ) + return super().transform_responses_api_request( + model=model, + input=normalized_input, + response_api_optional_request_params=response_api_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py index 2ea355fd369..a1e4345c3e1 100644 --- a/litellm/llms/bedrock_mantle/responses/transformation.py +++ b/litellm/llms/bedrock_mantle/responses/transformation.py @@ -15,14 +15,11 @@ role / access key / profile / web identity), signed via the shared BaseAWSLLM._sign_request after the request body is finalized. """ -import json -from collections.abc import Mapping from typing import Any, Final -from typing_extensions import ReadOnly, TypedDict - import litellm from litellm._logging import verbose_logger +from litellm.llms.base_llm.responses.codex_compat import normalize_codex_input_items from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock_mantle.common_utils import ( MANTLE_HOST_RE, @@ -54,33 +51,6 @@ _BEDROCK_MANTLE_SUPPORTED_SERVICE_TIERS: Final = frozenset({"auto", "default"}) _CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools" -_CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE: Final = "agent_message" -_CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE: Final = "context_compaction" -_CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE: Final = "local_shell_call" - - -class _RewrittenOutputTextBlock(TypedDict): - type: ReadOnly[str] - text: ReadOnly[str] - - -class _RewrittenAssistantMessageItem(TypedDict): - type: ReadOnly[str] - role: ReadOnly[str] - content: ReadOnly[tuple[_RewrittenOutputTextBlock, ...]] - - -class _RewrittenCompactionItem(TypedDict): - type: ReadOnly[str] - encrypted_content: ReadOnly[str] - - -class _RewrittenFunctionCallItem(TypedDict): - type: ReadOnly[str] - call_id: ReadOnly[str] - name: ReadOnly[str] - arguments: ReadOnly[str] - class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPIConfig): def __init__( @@ -186,7 +156,12 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI headers: dict, ) -> dict: remaining_input, hoisted_tools = self._hoist_codex_additional_tools(input) - normalized_input: Final = self._normalize_codex_input_items(remaining_input) + normalized_input, rewritten_types = normalize_codex_input_items(remaining_input) + if rewritten_types: + verbose_logger.warning( + "Bedrock Mantle Responses API: rewrote Codex input item type(s) %s that Mantle rejects.", + list(rewritten_types), + ) request_params: Final = ( { **response_api_optional_request_params, @@ -242,91 +217,6 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI ) return remaining_input, cls._filter_unsupported_tools(hoisted_tools) - @staticmethod - def _agent_message_text(item: "Mapping[str, object]") -> str: - content: Final = item.get("content") - if not isinstance(content, list): - return "" - return "".join( - str(block.get("text") or block.get("encrypted_content") or "") - for block in content - if isinstance(block, dict) - ) - - @classmethod - def _normalize_agent_message_item(cls, item: "Mapping[str, object]") -> "_RewrittenAssistantMessageItem | None": - text: Final = cls._agent_message_text(item) - if not text: - return None - rewritten: Final[_RewrittenAssistantMessageItem] = { - "type": "message", - "role": "assistant", - "content": ({"type": "output_text", "text": text},), - } - return rewritten - - @staticmethod - def _normalize_context_compaction_item(item: "Mapping[str, object]") -> "_RewrittenCompactionItem | None": - encrypted_content: Final = item.get("encrypted_content") - if not isinstance(encrypted_content, str) or not encrypted_content: - return None - rewritten: Final[_RewrittenCompactionItem] = {"type": "compaction", "encrypted_content": encrypted_content} - return rewritten - - @staticmethod - def _normalize_local_shell_call_item(item: "Mapping[str, object]") -> "_RewrittenFunctionCallItem | None": - call_id: Final = item.get("call_id") - if not isinstance(call_id, str) or not call_id: - return None - action: Final = item.get("action") - rewritten: Final[_RewrittenFunctionCallItem] = { - "type": "function_call", - "call_id": call_id, - "name": "local_shell", - "arguments": json.dumps(action) if isinstance(action, dict) else "{}", - } - return rewritten - - @classmethod - def _normalize_codex_input_item(cls, item: object) -> "tuple[object, str | None]": - """Returns (normalized item or None to drop it, original type when rewritten).""" - if not isinstance(item, dict): - return item, None - item_type: Final = item.get("type") - if item_type == _CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE: - return cls._normalize_agent_message_item(item), item_type - if item_type == _CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE: - return cls._normalize_context_compaction_item(item), item_type - if item_type == _CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE: - return cls._normalize_local_shell_call_item(item), item_type - return item, None - - @classmethod - def _normalize_codex_input_items( - cls, - input: "str | ResponseInputParam", - ) -> "str | ResponseInputParam": - """Rewrite Codex history item types Mantle rejects with 400 "Invalid - 'input': value did not match any expected variant" into supported - equivalents. `agent_message` (Codex multi-agent traffic; its - encrypted_content slot carries the plaintext payload when the model - never issued encrypted args) becomes an assistant message, - `context_compaction` becomes the `compaction` spelling Mantle accepts, - and `local_shell_call` becomes the function_call its recorded - function_call_output already pairs with. - """ - if not isinstance(input, list): - return input - normalized: Final = tuple(cls._normalize_codex_input_item(item) for item in input) - rewritten_types: Final = sorted(frozenset(item_type for _, item_type in normalized if item_type is not None)) - if rewritten_types: - verbose_logger.warning( - "Bedrock Mantle Responses API: rewrote Codex input item type(s) %s that Mantle rejects.", - rewritten_types, - ) - kept: Final = [item for item, _ in normalized if item is not None] # mutable-ok: ResponseInputParam is a list - return kept # pyright: ignore[reportReturnType] # Codex passthrough items sit outside the OpenAI input union - def map_openai_params( self, response_api_optional_params: ResponsesAPIOptionalRequestParams, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index dd367e875de..e144c0bb871 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -49672,7 +49672,10 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "supported_endpoints": [ + "/v1/responses" + ] }, "global.openai.gpt-5.6-sol": { "input_cost_per_token": 5e-06, @@ -49698,7 +49701,10 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "supported_endpoints": [ + "/v1/responses" + ] }, "us.openai.gpt-5.6-terra": { "input_cost_per_token": 2.2e-06, @@ -49724,7 +49730,10 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "supported_endpoints": [ + "/v1/responses" + ] }, "global.openai.gpt-5.6-terra": { "input_cost_per_token": 2e-06, @@ -49750,7 +49759,10 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "supported_endpoints": [ + "/v1/responses" + ] }, "us.openai.gpt-5.6-luna": { "input_cost_per_token": 2.2e-07, @@ -49776,7 +49788,10 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "supported_endpoints": [ + "/v1/responses" + ] }, "global.openai.gpt-5.6-luna": { "input_cost_per_token": 2e-07, @@ -49802,7 +49817,10 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "supported_endpoints": [ + "/v1/responses" + ] }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, diff --git a/litellm/utils.py b/litellm/utils.py index 54f97ccae54..b55f63cf272 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8593,6 +8593,11 @@ class ProviderConfigManager: return litellm.OpenRouterResponsesAPIConfig() elif litellm.LlmProviders.HOSTED_VLLM == provider: return litellm.HostedVLLMResponsesAPIConfig() + elif litellm.LlmProviders.BEDROCK == provider: + # bedrock-runtime serves the OpenAI models on an OpenAI-compatible surface + # (/openai/v1/responses) alongside Converse. The adapter decides whether a + # given model is on it; None keeps the chat-completions bridge. + return litellm.BedrockOpenAIResponsesConfig.for_model(model) elif litellm.LlmProviders.BEDROCK_MANTLE == provider: # Both decisions are data-driven from the model's price-map entry, with # no model-name logic. Capability (can it serve Responses?) comes from diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index dd367e875de..e144c0bb871 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -49672,7 +49672,10 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "supported_endpoints": [ + "/v1/responses" + ] }, "global.openai.gpt-5.6-sol": { "input_cost_per_token": 5e-06, @@ -49698,7 +49701,10 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "supported_endpoints": [ + "/v1/responses" + ] }, "us.openai.gpt-5.6-terra": { "input_cost_per_token": 2.2e-06, @@ -49724,7 +49730,10 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "supported_endpoints": [ + "/v1/responses" + ] }, "global.openai.gpt-5.6-terra": { "input_cost_per_token": 2e-06, @@ -49750,7 +49759,10 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "supported_endpoints": [ + "/v1/responses" + ] }, "us.openai.gpt-5.6-luna": { "input_cost_per_token": 2.2e-07, @@ -49776,7 +49788,10 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "supported_endpoints": [ + "/v1/responses" + ] }, "global.openai.gpt-5.6-luna": { "input_cost_per_token": 2e-07, @@ -49802,7 +49817,10 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "supported_endpoints": [ + "/v1/responses" + ] }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, diff --git a/tests/test_litellm/llms/base_llm/responses/test_codex_compat.py b/tests/test_litellm/llms/base_llm/responses/test_codex_compat.py new file mode 100644 index 00000000000..be811b17a82 --- /dev/null +++ b/tests/test_litellm/llms/base_llm/responses/test_codex_compat.py @@ -0,0 +1,116 @@ +"""Shared Codex wire-format normalization. + +Both Bedrock endpoints reject the Codex *history* item types with +``400 Invalid 'input': value did not match any expected variant``. They are history +items, so they only appear from the second turn of a session onward — a first-turn +smoke test passes and hides the problem entirely. +""" + +import json + +import pytest + +from litellm.llms.base_llm.responses.codex_compat import normalize_codex_input_items + +USER = {"role": "user", "content": "hi"} + + +class TestAgentMessage: + def test_becomes_an_assistant_message(self): + item = { + "type": "agent_message", + "role": "assistant", + "content": [{"type": "output_text", "text": "prior turn"}], + } + out, types = normalize_codex_input_items([item, USER]) + assert types == ("agent_message",) + assert out[0] == { + "type": "message", + "role": "assistant", + "content": ({"type": "output_text", "text": "prior turn"},), + } + + def test_encrypted_content_slot_is_used_as_text(self): + """Codex puts the plaintext payload there when the model issued no encrypted args.""" + item = {"type": "agent_message", "content": [{"encrypted_content": "plain"}]} + out, _ = normalize_codex_input_items([item, USER]) + assert out[0]["content"] == ({"type": "output_text", "text": "plain"},) + + def test_non_list_content_yields_no_text_and_drops_the_item(self): + out, types = normalize_codex_input_items([{"type": "agent_message", "content": "not a list"}, USER]) + assert out == [USER] + assert types == ("agent_message",) + + def test_textless_item_is_dropped(self): + out, types = normalize_codex_input_items([{"type": "agent_message", "content": []}, USER]) + assert out == [USER] + assert types == ("agent_message",) + + +class TestContextCompaction: + def test_becomes_compaction(self): + out, types = normalize_codex_input_items([{"type": "context_compaction", "encrypted_content": "abc"}, USER]) + assert out[0] == {"type": "compaction", "encrypted_content": "abc"} + assert types == ("context_compaction",) + + @pytest.mark.parametrize("bad", [{}, {"encrypted_content": ""}, {"encrypted_content": 7}]) + def test_without_usable_content_is_dropped(self, bad): + out, _ = normalize_codex_input_items([{"type": "context_compaction", **bad}, USER]) + assert out == [USER] + + +class TestLocalShellCall: + def test_becomes_the_function_call_its_output_pairs_with(self): + out, types = normalize_codex_input_items( + [{"type": "local_shell_call", "call_id": "c1", "action": {"command": ["ls"]}}, USER] + ) + assert out[0] == { + "type": "function_call", + "call_id": "c1", + "name": "local_shell", + "arguments": json.dumps({"command": ["ls"]}), + } + assert types == ("local_shell_call",) + + def test_missing_action_yields_empty_arguments(self): + out, _ = normalize_codex_input_items([{"type": "local_shell_call", "call_id": "c1"}, USER]) + assert out[0]["arguments"] == "{}" + + def test_without_call_id_is_dropped(self): + out, _ = normalize_codex_input_items([{"type": "local_shell_call"}, USER]) + assert out == [USER] + + +class TestPassthroughAndShape: + def test_string_input_untouched(self): + assert normalize_codex_input_items("just a prompt") == ("just a prompt", ()) + + def test_unrelated_items_untouched_and_no_types_reported(self): + items = [USER, {"type": "message", "role": "assistant", "content": []}] + out, types = normalize_codex_input_items(items) + assert out == items + assert types == () + + def test_non_mapping_entries_pass_through_except_a_literal_none(self): + """A literal ``None`` is indistinguishable from "drop this item" in the + per-item return protocol, so it is dropped. Other non-mapping entries pass + through untouched. This matches the behaviour before the normalizer moved + out of the bedrock_mantle config.""" + out, types = normalize_codex_input_items(["a string", 42, None, USER]) + assert out == ["a string", 42, USER] + assert types == () + + def test_types_are_sorted_and_deduplicated(self): + items = [ + {"type": "local_shell_call", "call_id": "c1"}, + {"type": "agent_message", "content": [{"text": "x"}]}, + {"type": "local_shell_call", "call_id": "c2"}, + ] + _, types = normalize_codex_input_items(items) + assert types == ("agent_message", "local_shell_call") + + def test_returns_a_list_not_a_tuple(self): + """The input->messages conversion downstream narrows on isinstance(input, list); + a tuple silently yields zero messages and the provider rejects the request.""" + out, _ = normalize_codex_input_items([{"type": "agent_message", "content": [{"text": "x"}]}, USER]) + assert isinstance(out, list) diff --git a/tests/test_litellm/llms/bedrock/responses/test_bedrock_openai_responses.py b/tests/test_litellm/llms/bedrock/responses/test_bedrock_openai_responses.py new file mode 100644 index 00000000000..b256eaf8f88 --- /dev/null +++ b/tests/test_litellm/llms/bedrock/responses/test_bedrock_openai_responses.py @@ -0,0 +1,211 @@ +"""Native OpenAI Responses API on the bedrock-runtime endpoint. + +Without this config the bedrock provider has no Responses config, so /v1/responses +falls back to the Chat Completions bridge and rides Converse. +""" + +import json +from importlib.resources import files +from unittest.mock import patch + +import pytest + +import litellm +from litellm.llms.bedrock.common_utils import bedrock_supports_openai_responses +from litellm.llms.bedrock.responses.transformation import BedrockOpenAIResponsesConfig +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + +MODEL = "global.openai.gpt-5.6-sol" + + +def _cfg(): + return BedrockOpenAIResponsesConfig() + + +class TestCompleteURL: + def test_default_host_and_path(self): + url = _cfg().get_complete_url(None, {"aws_region_name": "us-east-1"}) + assert url == "https://bedrock-runtime.us-east-1.amazonaws.com/openai/v1/responses" + + def test_region_is_honoured(self): + url = _cfg().get_complete_url(None, {"aws_region_name": "eu-west-1"}) + assert url == "https://bedrock-runtime.eu-west-1.amazonaws.com/openai/v1/responses" + + @pytest.mark.parametrize( + "api_base", + [ + "https://proxy.example.com", + "https://proxy.example.com/", + "https://proxy.example.com/openai/v1", + "https://proxy.example.com/openai/v1/responses", + ], + ) + def test_custom_host_is_preserved_and_path_never_doubles(self, api_base): + url = _cfg().get_complete_url(api_base, {"aws_region_name": "us-east-1"}) + assert url == "https://proxy.example.com/openai/v1/responses" + + def test_runtime_endpoint_param_is_honoured(self): + url = _cfg().get_complete_url( + None, {"aws_region_name": "us-east-1", "aws_bedrock_runtime_endpoint": "https://vpce.example.com"} + ) + assert url == "https://vpce.example.com/openai/v1/responses" + + +class TestAuth: + def test_bearer_token_is_used_when_present(self): + headers = _cfg().validate_environment({}, MODEL, GenericLiteLLMParams(api_key="sk-bedrock")) + assert headers["Authorization"] == "Bearer sk-bedrock" + + def test_no_authorization_header_without_a_token(self, monkeypatch): + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + headers = _cfg().validate_environment({}, MODEL, GenericLiteLLMParams()) + assert "Authorization" not in headers + + def test_sigv4_is_skipped_when_a_bearer_token_is_present(self): + """Bedrock API keys are Bearer; signing on top would be wrong.""" + headers, body = _cfg().sign_request( + headers={"Authorization": "Bearer sk-bedrock"}, + optional_params={}, + request_data={}, + api_base="https://bedrock-runtime.us-east-1.amazonaws.com/openai/v1/responses", + api_key="sk-bedrock", + ) + assert headers["Authorization"] == "Bearer sk-bedrock" + assert body is None + + +class TestProviderIdentity: + def test_reports_the_bedrock_provider(self): + """Cost tracking and callbacks key off this, so it must stay `bedrock` rather + than becoming a separate provider.""" + assert _cfg().custom_llm_provider == LlmProviders.BEDROCK + + +class TestSigV4Fallback: + def test_signs_with_sigv4_when_no_bearer_token_is_present(self, monkeypatch): + """No Bedrock API key means SigV4 over the standard credential chain. Static + credentials are set in the environment so signing stays a local computation.""" + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY") + monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + headers, body = _cfg().sign_request( + headers={"content-type": "application/json"}, + optional_params={"aws_region_name": "us-east-1"}, + request_data={"model": MODEL, "input": "hi"}, + api_base="https://bedrock-runtime.us-east-1.amazonaws.com/openai/v1/responses", + api_key=None, + ) + assert "Authorization" in headers + assert headers["Authorization"].startswith("AWS4-HMAC-SHA256") + assert "Credential=AKIAIOSFODNN7EXAMPLE" in headers["Authorization"] + + +class TestPriceMapGate: + def test_absent_model_has_no_signal(self): + assert bedrock_supports_openai_responses(MODEL, {}) is False + + def test_none_model_is_false(self): + assert bedrock_supports_openai_responses(None, {}) is False + + def test_signal_on_the_bare_key(self): + cost = {MODEL: {"supported_endpoints": ["/v1/responses"]}} + assert bedrock_supports_openai_responses(MODEL, cost) is True + + def test_signal_on_the_bedrock_prefixed_key(self): + cost = {f"bedrock/{MODEL}": {"supported_endpoints": ["/v1/responses"]}} + assert bedrock_supports_openai_responses(MODEL, cost) is True + + def test_other_endpoints_do_not_count(self): + cost = {MODEL: {"supported_endpoints": ["/v1/messages"]}} + assert bedrock_supports_openai_responses(MODEL, cost) is False + + +class TestForModelGate: + """The capability decision lives on the adapter, not in the shared dispatch.""" + + def test_returns_a_config_for_a_signalled_model(self): + with patch.object( # test-quality-ok: the gate reads the global cost map by design; no injection point exists + litellm, "model_cost", {MODEL: {"supported_endpoints": ["/v1/responses"]}} + ): + assert isinstance(BedrockOpenAIResponsesConfig.for_model(MODEL), BedrockOpenAIResponsesConfig) + + def test_returns_none_for_an_unsignalled_model(self): + with patch.object( # test-quality-ok: the gate reads the global cost map by design; no injection point exists + litellm, "model_cost", {} + ): + assert BedrockOpenAIResponsesConfig.for_model(MODEL) is None + + def test_returns_none_for_no_model(self): + with patch.object( # test-quality-ok: the gate reads the global cost map by design; no injection point exists + litellm, "model_cost", {} + ): + assert BedrockOpenAIResponsesConfig.for_model(None) is None + + +class TestProviderResolution: + """model_cost is patched explicitly: it is populated at import time from a GitHub + fetch unless LITELLM_LOCAL_MODEL_COST_MAP is set, and conftest's monkeypatch of + that variable lands after import — so these must not read the global.""" + + def test_signalled_model_resolves_to_the_bedrock_responses_config(self): + with patch.object( # test-quality-ok: resolution reads the global cost map by design; no HTTP boundary or injection point exists + litellm, "model_cost", {MODEL: {"supported_endpoints": ["/v1/responses"]}} + ): + cfg = ProviderConfigManager.get_provider_responses_api_config(model=MODEL, provider=LlmProviders.BEDROCK) + assert isinstance(cfg, BedrockOpenAIResponsesConfig) + + def test_unsignalled_model_keeps_the_existing_bridge(self): + """Claude on Bedrock has no OpenAI surface; it must keep falling through to + the chat-completions bridge exactly as before.""" + with patch.object(litellm, "model_cost", {}): # test-quality-ok: resolution reads the global cost map by design + cfg = ProviderConfigManager.get_provider_responses_api_config( + model="anthropic.claude-3-haiku-20240307-v1:0", provider=LlmProviders.BEDROCK + ) + assert cfg is None + + def test_the_shipped_price_map_signals_the_gpt_56_family(self): + """Reads the bundled backup directly rather than the network-fetched global.""" + shipped = json.loads( + files("litellm").joinpath("model_prices_and_context_window_backup.json").read_text(encoding="utf-8") + ) + for prefix in ("us", "global"): + for variant in ("sol", "terra", "luna"): + model = f"{prefix}.openai.gpt-5.6-{variant}" + assert bedrock_supports_openai_responses(model, shipped) is True, model + + +class TestCodexHistoryNormalization: + def test_history_items_the_endpoint_rejects_are_rewritten(self): + body = _cfg().transform_responses_api_request( + model=MODEL, + input=[ + {"type": "agent_message", "content": [{"type": "output_text", "text": "prior"}]}, + {"type": "context_compaction", "encrypted_content": "abc"}, + {"type": "local_shell_call", "call_id": "c1", "action": {"command": ["ls"]}}, + {"role": "user", "content": "carry on"}, + ], + response_api_optional_request_params={}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert [i.get("type") or i.get("role") for i in body["input"]] == [ + "message", + "compaction", + "function_call", + "user", + ] + + def test_a_first_turn_request_is_untouched(self): + """The rejected types are history items, so turn one exercises none of this.""" + original = [{"role": "user", "content": "first turn"}] + body = _cfg().transform_responses_api_request( + model=MODEL, + input=list(original), + response_api_optional_request_params={}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert body["input"] == original diff --git a/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py b/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py index 1388073381e..3b0de4bdd2b 100644 --- a/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py +++ b/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py @@ -287,9 +287,11 @@ def test_bedrock_gpt_5_6_advertises_only_converse_supported_features( raw = _packaged_cost_map()[profile.model_id] assert raw["supported_modalities"] == ["text", "image"] assert raw["supported_output_modalities"] == ["text"] - # No bedrock_converse entry declares supported_endpoints; these models are reachable - # on chat completions and on the Responses API without it. - assert "supported_endpoints" not in raw + # supported_endpoints opts these models into bedrock-runtime's native OpenAI + # Responses surface (/openai/v1/responses), which AWS serves alongside Converse. + # It is the only signal that selects it; without the entry they fall back to the + # Chat Completions bridge, as every other bedrock_converse model still does. + assert raw["supported_endpoints"] == ["/v1/responses"] @pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id)