mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-26 01:12:21 +00:00
fix(bedrock): route gpt-oss response_format to Converse and decide the route once from the raw request
This commit is contained in:
mateo-berri
parent
6b9f067c80
commit
a1c089f107
12 changed files with 248 additions and 65 deletions
|
|
@ -30,8 +30,6 @@ EXTRA_BOOLEAN_KEYS = frozenset(
|
|||
"gemini_audio_only_live",
|
||||
"uses_embed_content",
|
||||
"use_openai_responses_path",
|
||||
"use_bedrock_runtime_chat_completions",
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none",
|
||||
"bedrock_converse_supports_strict_tools",
|
||||
"thinking_always_on",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ Native OpenAI Chat Completions on Amazon Bedrock Runtime.
|
|||
|
||||
AWS serves this surface at
|
||||
``https://bedrock-runtime.{region}.amazonaws.com/openai/v1/chat/completions``
|
||||
for the models whose price-map entry sets ``use_bedrock_runtime_chat_completions``
|
||||
for the models whose price-map entry sets ``supports_bedrock_runtime_chat_completions``
|
||||
(Grok 4.6, gpt-oss, the GPT-5.6 family): chat completions stay chat completions
|
||||
instead of being rewritten to Converse.
|
||||
|
||||
|
|
@ -19,6 +19,7 @@ from types import MappingProxyType
|
|||
from typing import TYPE_CHECKING, Final, Literal
|
||||
|
||||
import httpx
|
||||
from typing_extensions import assert_never
|
||||
|
||||
import litellm
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
|
@ -72,6 +73,8 @@ class ReasoningTagSplitter:
|
|||
return self._feed_start(self.pending + text)
|
||||
case "reasoning":
|
||||
return self._feed_reasoning(self.pending + text)
|
||||
case _:
|
||||
assert_never(self.phase)
|
||||
|
||||
def _feed_start(self, buffered: str) -> tuple["ReasoningTagSplitter", str, str]:
|
||||
if buffered.startswith(REASONING_OPEN_TAG):
|
||||
|
|
|
|||
|
|
@ -803,22 +803,33 @@ def _bedrock_price_map_flag(model: str, flag: str) -> bool:
|
|||
def uses_bedrock_runtime_chat_completions(model: str) -> bool:
|
||||
"""Whether this Bedrock model should use runtime native Chat Completions.
|
||||
|
||||
Data-driven from the price-map ``use_bedrock_runtime_chat_completions`` flag
|
||||
Data-driven from the price-map ``supports_bedrock_runtime_chat_completions`` flag
|
||||
so onboarding a model is a JSON change. Explicit ``converse/`` still wins in
|
||||
``get_bedrock_route`` because prefix routes are checked first, and a request
|
||||
that needs a Converse-only feature (``bedrock_request_needs_converse``) is
|
||||
served by Converse even on a flagged model.
|
||||
"""
|
||||
return _bedrock_price_map_flag(model, "use_bedrock_runtime_chat_completions")
|
||||
return _bedrock_price_map_flag(model, "supports_bedrock_runtime_chat_completions")
|
||||
|
||||
|
||||
def bedrock_runtime_chat_completions_tools_require_reasoning_none(model: str) -> bool:
|
||||
"""Whether AWS's native Chat Completions only serves this model's function tools with ``reasoning_effort="none"``.
|
||||
def bedrock_runtime_chat_completions_serves_tools_with_reasoning(model: str) -> bool:
|
||||
"""Whether AWS's native Chat Completions serves this model's function tools with any ``reasoning_effort``.
|
||||
|
||||
Data-driven from the price-map ``bedrock_runtime_chat_completions_tools_require_reasoning_none``
|
||||
flag (the GPT-5.6 family). Converse serves tools with any effort, so those requests fall back to it.
|
||||
Data-driven from the price-map ``supports_bedrock_runtime_chat_completions_tools_with_reasoning``
|
||||
flag (gpt-oss, Grok). Without it AWS only takes tools with ``reasoning_effort="none"``
|
||||
(the GPT-5.6 family), and Converse serves tools with any effort, so those requests fall back to it.
|
||||
"""
|
||||
return _bedrock_price_map_flag(model, "bedrock_runtime_chat_completions_tools_require_reasoning_none")
|
||||
return _bedrock_price_map_flag(model, "supports_bedrock_runtime_chat_completions_tools_with_reasoning")
|
||||
|
||||
|
||||
def bedrock_runtime_chat_completions_enforces_response_format(model: str) -> bool:
|
||||
"""Whether AWS's native Chat Completions enforces a ``response_format`` schema for this model.
|
||||
|
||||
Data-driven from the price-map ``supports_bedrock_runtime_chat_completions_response_format`` flag
|
||||
(GPT-5.6, Grok). Without it AWS accepts the field and answers with unconstrained text (gpt-oss), so
|
||||
Converse, which emulates the schema through a forced ``json_tool_call`` tool, serves those requests.
|
||||
"""
|
||||
return _bedrock_price_map_flag(model, "supports_bedrock_runtime_chat_completions_response_format")
|
||||
|
||||
|
||||
BEDROCK_CONVERSE_ONLY_REQUEST_KEYS: Final = frozenset(
|
||||
|
|
@ -826,26 +837,52 @@ BEDROCK_CONVERSE_ONLY_REQUEST_KEYS: Final = frozenset(
|
|||
)
|
||||
|
||||
|
||||
def _response_format_constrains_output(response_format: object) -> bool:
|
||||
if response_format is None:
|
||||
return False
|
||||
return not (isinstance(response_format, Mapping) and response_format.get("type") == "text")
|
||||
|
||||
|
||||
def bedrock_request_needs_converse(model: str, request_params: Mapping[str, object]) -> bool:
|
||||
"""Whether a request on a runtime-Chat-Completions model must still be served by Converse.
|
||||
|
||||
Converse-shaped body keys (``BEDROCK_CONVERSE_ONLY_REQUEST_KEYS``) are rejected as malformed input by
|
||||
AWS's native OpenAI surface, operator-owned request metadata is only written onto the Converse body,
|
||||
and function tools on a ``bedrock_runtime_chat_completions_tools_require_reasoning_none`` model are
|
||||
rejected there unless ``reasoning_effort`` is exactly ``"none"``.
|
||||
function tools on a model without ``supports_bedrock_runtime_chat_completions_tools_with_reasoning``
|
||||
are rejected there unless ``reasoning_effort`` is exactly ``"none"``, and a constraining
|
||||
``response_format`` on a model without ``supports_bedrock_runtime_chat_completions_response_format``
|
||||
is only honored by Converse.
|
||||
"""
|
||||
if any(request_params.get(key) is not None for key in BEDROCK_CONVERSE_ONLY_REQUEST_KEYS):
|
||||
return True
|
||||
if bedrock_request_metadata_is_owned():
|
||||
return True
|
||||
if _response_format_constrains_output(
|
||||
request_params.get("response_format")
|
||||
) and not bedrock_runtime_chat_completions_enforces_response_format(model):
|
||||
return True
|
||||
if not request_params.get("tools"):
|
||||
return False
|
||||
return (
|
||||
bedrock_runtime_chat_completions_tools_require_reasoning_none(model)
|
||||
not bedrock_runtime_chat_completions_serves_tools_with_reasoning(model)
|
||||
and request_params.get("reasoning_effort") != "none"
|
||||
)
|
||||
|
||||
|
||||
def bedrock_route_for_request(
|
||||
model: str, request_params: Mapping[str, object], additional_drop_params: Sequence[str] | None
|
||||
) -> BedrockRoute:
|
||||
"""The route for one request, decided from the caller's raw params before any provider mapping.
|
||||
|
||||
Param mapping and dispatch both call this with the same inputs, so a request that falls back to
|
||||
Converse is mapped with the Converse config and sent to Converse, never one without the other.
|
||||
"""
|
||||
dropped: Final = frozenset(additional_drop_params or ())
|
||||
return BedrockModelInfo.get_bedrock_route(
|
||||
model, {key: value for key, value in request_params.items() if key not in dropped}
|
||||
)
|
||||
|
||||
|
||||
def strip_bedrock_throughput_suffix(model: str) -> str:
|
||||
"""Strip throughput tier suffixes and context window suffixes from Bedrock model names."""
|
||||
import re
|
||||
|
|
|
|||
|
|
@ -104,7 +104,7 @@ from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig
|
|||
from litellm.llms.base_llm.base_model_iterator import (
|
||||
convert_model_response_to_streaming,
|
||||
)
|
||||
from litellm.llms.bedrock.common_utils import BedrockModelInfo
|
||||
from litellm.llms.bedrock.common_utils import BedrockModelInfo, bedrock_route_for_request
|
||||
from litellm.llms.cohere.common_utils import CohereModelInfo
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled
|
||||
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
|
||||
|
|
@ -4078,7 +4078,9 @@ def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes
|
|||
if "aws_region_name" not in optional_params or optional_params["aws_region_name"] is None:
|
||||
optional_params["aws_region_name"] = aws_bedrock_client.meta.region_name
|
||||
|
||||
bedrock_route: Final = BedrockModelInfo.get_bedrock_route(model, optional_params)
|
||||
bedrock_route: Final = bedrock_route_for_request(
|
||||
model, ctx.request_params, ctx.kwargs.get("additional_drop_params")
|
||||
)
|
||||
if bedrock_route == "claude_platform":
|
||||
provider_config = ProviderConfigManager.get_provider_chat_config(
|
||||
model=model,
|
||||
|
|
@ -5686,6 +5688,7 @@ def completion(
|
|||
optional_params=optional_params,
|
||||
organization=organization,
|
||||
provider_config=provider_config,
|
||||
request_params={**optional_param_args, **non_default_params},
|
||||
shared_session=shared_session,
|
||||
stream=stream,
|
||||
temperature=temperature,
|
||||
|
|
|
|||
|
|
@ -40870,7 +40870,8 @@
|
|||
"output_cost_per_token": 0.0
|
||||
},
|
||||
"openai.gpt-oss-120b-1:0": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -40884,7 +40885,8 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"openai.gpt-oss-20b-1:0": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
|
||||
"input_cost_per_token": 7e-08,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -47234,7 +47236,9 @@
|
|||
"input_cost_per_token": 2.64e-06,
|
||||
"output_cost_per_token": 7.92e-06,
|
||||
"cache_read_input_token_cost": 6.6e-07,
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 500000,
|
||||
"max_output_tokens": 500000,
|
||||
|
|
@ -58443,8 +58447,8 @@
|
|||
"supports_web_search": true
|
||||
},
|
||||
"us.openai.gpt-5.6-sol": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"input_cost_per_token": 4.4e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 8.8e-06,
|
||||
"cache_creation_input_token_cost": 5.5e-06,
|
||||
|
|
@ -58474,8 +58478,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"global.openai.gpt-5.6-sol": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"input_cost_per_token": 4e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 8e-06,
|
||||
"cache_creation_input_token_cost": 5e-06,
|
||||
|
|
@ -58505,8 +58509,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"us.openai.gpt-5.6-terra": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"input_cost_per_token": 2.2e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 4.4e-06,
|
||||
"cache_creation_input_token_cost": 2.75e-06,
|
||||
|
|
@ -58536,8 +58540,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"global.openai.gpt-5.6-terra": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 4e-06,
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
|
|
@ -58567,8 +58571,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"us.openai.gpt-5.6-luna": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"input_cost_per_token": 2.2e-07,
|
||||
"input_cost_per_token_above_272k_tokens": 4.4e-07,
|
||||
"cache_creation_input_token_cost": 2.75e-07,
|
||||
|
|
@ -58598,8 +58602,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"global.openai.gpt-5.6-luna": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"input_cost_per_token_above_272k_tokens": 4e-07,
|
||||
"cache_creation_input_token_cost": 2.5e-07,
|
||||
|
|
@ -58909,7 +58913,9 @@
|
|||
"input_cost_per_token": 2.2e-06,
|
||||
"output_cost_per_token": 6.6e-06,
|
||||
"cache_read_input_token_cost": 5.5e-07,
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 500000,
|
||||
"max_output_tokens": 500000,
|
||||
|
|
@ -58925,7 +58931,9 @@
|
|||
"input_cost_per_token": 2e-06,
|
||||
"output_cost_per_token": 6e-06,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 500000,
|
||||
"max_output_tokens": 500000,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Coroutine, Iterable
|
||||
from collections.abc import Callable, Coroutine, Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Literal, Union
|
||||
|
||||
|
|
@ -229,6 +229,7 @@ class _CompletionDispatchContext:
|
|||
optional_params: dict
|
||||
organization: str | None
|
||||
provider_config: BaseConfig | None
|
||||
request_params: Mapping[str, object]
|
||||
shared_session: ClientSession | None
|
||||
stream: bool | None
|
||||
temperature: float | None
|
||||
|
|
|
|||
|
|
@ -401,7 +401,7 @@ if TYPE_CHECKING:
|
|||
BaseVectorStoreFilesConfig,
|
||||
)
|
||||
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
|
||||
from litellm.llms.bedrock.common_utils import BedrockModelInfo, BedrockRoute
|
||||
from litellm.llms.bedrock.common_utils import BedrockRoute
|
||||
from litellm.llms.bedrock.embed.amazon_nova_transformation import (
|
||||
AmazonNovaEmbeddingConfig,
|
||||
)
|
||||
|
|
@ -3353,12 +3353,9 @@ def _should_drop_param(k, additional_drop_params) -> bool:
|
|||
def _bedrock_route_for_request(
|
||||
model: str, passed_params: Mapping[str, object], additional_drop_params: list | None
|
||||
) -> BedrockRoute:
|
||||
from litellm.llms.bedrock.common_utils import BedrockModelInfo
|
||||
from litellm.llms.bedrock.common_utils import bedrock_route_for_request
|
||||
|
||||
return BedrockModelInfo.get_bedrock_route(
|
||||
model,
|
||||
{k: v for k, v in passed_params.items() if not _should_drop_param(k, additional_drop_params)},
|
||||
)
|
||||
return bedrock_route_for_request(model, passed_params, additional_drop_params)
|
||||
|
||||
|
||||
def _get_non_default_params(passed_params: dict, default_params: dict, additional_drop_params: list | None) -> dict:
|
||||
|
|
|
|||
|
|
@ -40870,7 +40870,8 @@
|
|||
"output_cost_per_token": 0.0
|
||||
},
|
||||
"openai.gpt-oss-120b-1:0": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -40884,7 +40885,8 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"openai.gpt-oss-20b-1:0": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
|
||||
"input_cost_per_token": 7e-08,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -47234,7 +47236,9 @@
|
|||
"input_cost_per_token": 2.64e-06,
|
||||
"output_cost_per_token": 7.92e-06,
|
||||
"cache_read_input_token_cost": 6.6e-07,
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 500000,
|
||||
"max_output_tokens": 500000,
|
||||
|
|
@ -58443,8 +58447,8 @@
|
|||
"supports_web_search": true
|
||||
},
|
||||
"us.openai.gpt-5.6-sol": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"input_cost_per_token": 4.4e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 8.8e-06,
|
||||
"cache_creation_input_token_cost": 5.5e-06,
|
||||
|
|
@ -58474,8 +58478,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"global.openai.gpt-5.6-sol": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"input_cost_per_token": 4e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 8e-06,
|
||||
"cache_creation_input_token_cost": 5e-06,
|
||||
|
|
@ -58505,8 +58509,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"us.openai.gpt-5.6-terra": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"input_cost_per_token": 2.2e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 4.4e-06,
|
||||
"cache_creation_input_token_cost": 2.75e-06,
|
||||
|
|
@ -58536,8 +58540,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"global.openai.gpt-5.6-terra": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 4e-06,
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
|
|
@ -58567,8 +58571,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"us.openai.gpt-5.6-luna": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"input_cost_per_token": 2.2e-07,
|
||||
"input_cost_per_token_above_272k_tokens": 4.4e-07,
|
||||
"cache_creation_input_token_cost": 2.75e-07,
|
||||
|
|
@ -58598,8 +58602,8 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"global.openai.gpt-5.6-luna": {
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"input_cost_per_token_above_272k_tokens": 4e-07,
|
||||
"cache_creation_input_token_cost": 2.5e-07,
|
||||
|
|
@ -58909,7 +58913,9 @@
|
|||
"input_cost_per_token": 2.2e-06,
|
||||
"output_cost_per_token": 6.6e-06,
|
||||
"cache_read_input_token_cost": 5.5e-07,
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 500000,
|
||||
"max_output_tokens": 500000,
|
||||
|
|
@ -58925,7 +58931,9 @@
|
|||
"input_cost_per_token": 2e-06,
|
||||
"output_cost_per_token": 6e-06,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"use_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions": true,
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 500000,
|
||||
"max_output_tokens": 500000,
|
||||
|
|
|
|||
|
|
@ -74,9 +74,6 @@
|
|||
"xhigh"
|
||||
]
|
||||
},
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"cache_creation_input_audio_token_cost": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
|
|
@ -830,6 +827,15 @@
|
|||
"supports_audio_output": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_bedrock_runtime_chat_completions": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_bedrock_runtime_chat_completions_response_format": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_computer_use": {
|
||||
"type": "boolean"
|
||||
},
|
||||
|
|
@ -1000,9 +1006,6 @@
|
|||
"minimum": 0,
|
||||
"description": "Provider default tokens-per-minute limit."
|
||||
},
|
||||
"use_bedrock_runtime_chat_completions": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"use_openai_responses_path": {
|
||||
"type": "boolean"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import json
|
|||
|
||||
import httpx
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
|
||||
import litellm
|
||||
from litellm.llms.bedrock.chat.chat_completions.transformation import (
|
||||
|
|
@ -17,6 +18,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
BEDROCK_CONVERSE_ONLY_REQUEST_KEYS,
|
||||
BedrockModelInfo,
|
||||
bedrock_request_needs_converse,
|
||||
bedrock_route_for_request,
|
||||
get_bedrock_chat_config,
|
||||
uses_bedrock_runtime_chat_completions,
|
||||
)
|
||||
|
|
@ -471,7 +473,7 @@ def test_converse_only_request_keys_go_to_converse(local_cost_map, fake_aws_env,
|
|||
)
|
||||
|
||||
assert requests[0].url.raw_path.endswith(b"/model/openai.gpt-oss-20b-1%3A0/converse")
|
||||
(key, value), = converse_only_param.items()
|
||||
((key, value),) = converse_only_param.items()
|
||||
assert json.loads(requests[0].content)[key] == value
|
||||
|
||||
|
||||
|
|
@ -620,3 +622,124 @@ def test_streaming_handler_keeps_native_reasoning_next_to_the_tagged_split():
|
|||
|
||||
assert parsed.choices[0].delta.reasoning_content == "native tagged"
|
||||
assert parsed.choices[0].delta.content == "Hi"
|
||||
|
||||
|
||||
RESPONSE_FORMAT_JSON_SCHEMA = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "answer",
|
||||
"schema": {"type": "object", "properties": {"word": {"type": "string"}}, "required": ["word"]},
|
||||
"strict": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class Answer(BaseModel):
|
||||
word: str
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["openai.gpt-oss-20b-1:0", "bedrock/openai.gpt-oss-120b-1:0"])
|
||||
@pytest.mark.parametrize(
|
||||
"response_format, expected_route",
|
||||
[
|
||||
(RESPONSE_FORMAT_JSON_SCHEMA, "converse"),
|
||||
({"type": "json_object"}, "converse"),
|
||||
(Answer, "converse"),
|
||||
({"type": "text"}, "chat_completions"),
|
||||
(None, "chat_completions"),
|
||||
],
|
||||
ids=["json_schema", "json_object", "pydantic", "text", "none"],
|
||||
)
|
||||
def test_gpt_oss_response_format_falls_back_to_converse(local_cost_map, model, response_format, expected_route):
|
||||
params = {"response_format": response_format}
|
||||
assert bedrock_request_needs_converse(model, params) is (expected_route == "converse")
|
||||
assert BedrockModelInfo.get_bedrock_route(model, params) == expected_route
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["global.openai.gpt-5.6-sol", "us.xai.grok-4.6", "bedrock/us-gov.xai.grok-4.6"])
|
||||
def test_response_format_stays_on_chat_completions_where_aws_enforces_it(local_cost_map, model):
|
||||
params = {"response_format": RESPONSE_FORMAT_JSON_SCHEMA}
|
||||
assert bedrock_request_needs_converse(model, params) is False
|
||||
assert BedrockModelInfo.get_bedrock_route(model, params) == "chat_completions"
|
||||
|
||||
|
||||
SYNTHETIC_NATIVE_MODEL = "vendor.native-model-v1:0"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"capability_flags, request_params, needs_converse",
|
||||
[
|
||||
({}, {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"}, True),
|
||||
({}, {"tools": [GET_WEATHER_TOOL]}, True),
|
||||
({}, {"tools": [GET_WEATHER_TOOL], "reasoning_effort": "none"}, False),
|
||||
(
|
||||
{"supports_bedrock_runtime_chat_completions_tools_with_reasoning": True},
|
||||
{"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"},
|
||||
False,
|
||||
),
|
||||
({}, {"response_format": RESPONSE_FORMAT_JSON_SCHEMA}, True),
|
||||
(
|
||||
{"supports_bedrock_runtime_chat_completions_response_format": True},
|
||||
{"response_format": RESPONSE_FORMAT_JSON_SCHEMA},
|
||||
False,
|
||||
),
|
||||
(
|
||||
{"supports_bedrock_runtime_chat_completions_response_format": True},
|
||||
{"response_format": RESPONSE_FORMAT_JSON_SCHEMA, "tools": [GET_WEATHER_TOOL], "reasoning_effort": "low"},
|
||||
True,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_capability_flags_are_read_from_the_cost_map(monkeypatch, capability_flags, request_params, needs_converse):
|
||||
entry = {
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"supports_bedrock_runtime_chat_completions": True,
|
||||
**capability_flags,
|
||||
}
|
||||
monkeypatch.setattr(litellm, "model_cost", {SYNTHETIC_NATIVE_MODEL: entry})
|
||||
assert bedrock_request_needs_converse(SYNTHETIC_NATIVE_MODEL, request_params) is needs_converse
|
||||
route = bedrock_route_for_request(SYNTHETIC_NATIVE_MODEL, request_params, None)
|
||||
assert (route == "chat_completions") is (not needs_converse)
|
||||
|
||||
|
||||
def test_route_for_request_ignores_dropped_params(local_cost_map):
|
||||
params = {"response_format": RESPONSE_FORMAT_JSON_SCHEMA, "guardrailConfig": {"guardrailIdentifier": "gr-1"}}
|
||||
assert bedrock_route_for_request("openai.gpt-oss-20b-1:0", params, None) == "converse"
|
||||
assert bedrock_route_for_request("openai.gpt-oss-20b-1:0", params, ["guardrailConfig"]) == "converse"
|
||||
assert (
|
||||
bedrock_route_for_request("openai.gpt-oss-20b-1:0", params, ["guardrailConfig", "response_format"])
|
||||
== "chat_completions"
|
||||
)
|
||||
|
||||
|
||||
def test_gpt_oss_response_format_goes_to_converse_with_json_tool_call(local_cost_map, fake_aws_env):
|
||||
requests, client = _recording_client(json=CONVERSE_JSON)
|
||||
litellm.completion(
|
||||
model="bedrock/openai.gpt-oss-20b-1:0",
|
||||
messages=[{"role": "user", "content": "Reply with the single word pong."}],
|
||||
response_format=RESPONSE_FORMAT_JSON_SCHEMA,
|
||||
max_tokens=64,
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert requests[0].url.raw_path.endswith(b"/model/openai.gpt-oss-20b-1%3A0/converse")
|
||||
body = json.loads(requests[0].content)
|
||||
assert body["toolConfig"]["tools"][0]["toolSpec"]["name"] == "json_tool_call"
|
||||
assert body["toolConfig"]["toolChoice"] == {"tool": {"name": "json_tool_call"}}
|
||||
assert body["inferenceConfig"]["maxTokens"] == 64
|
||||
assert "response_format" not in body
|
||||
assert "max_completion_tokens" not in body
|
||||
|
||||
|
||||
def test_gpt56_response_format_is_sent_as_is_on_chat_completions(local_cost_map, fake_aws_env):
|
||||
requests, client = _recording_client(json=_chat_completion_json('{"word": "pong"}', "global.openai.gpt-5.6-sol"))
|
||||
response = litellm.completion(
|
||||
model="bedrock/global.openai.gpt-5.6-sol",
|
||||
messages=[{"role": "user", "content": "Reply with the single word pong."}],
|
||||
response_format=RESPONSE_FORMAT_JSON_SCHEMA,
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1/chat/completions"
|
||||
assert json.loads(requests[0].content)["response_format"] == RESPONSE_FORMAT_JSON_SCHEMA
|
||||
assert response.choices[0].message.content == '{"word": "pong"}'
|
||||
|
|
|
|||
|
|
@ -878,8 +878,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
|
|||
"supports_video_input": {"type": "boolean"},
|
||||
"supports_vision": {"type": "boolean"},
|
||||
"supports_web_search": {"type": "boolean"},
|
||||
"use_bedrock_runtime_chat_completions": {"type": "boolean"},
|
||||
"bedrock_runtime_chat_completions_tools_require_reasoning_none": {"type": "boolean"},
|
||||
"supports_bedrock_runtime_chat_completions": {"type": "boolean"},
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": {"type": "boolean"},
|
||||
"supports_bedrock_runtime_chat_completions_response_format": {"type": "boolean"},
|
||||
"supports_url_context": {"type": "boolean"},
|
||||
"supports_multimodal": {"type": "boolean"},
|
||||
"uses_embed_content": {"type": "boolean"},
|
||||
|
|
|
|||
|
|
@ -181,6 +181,7 @@ def _build_dispatch_context() -> _CompletionDispatchContext:
|
|||
optional_params={},
|
||||
organization=None,
|
||||
provider_config=None,
|
||||
request_params={},
|
||||
shared_session=None,
|
||||
stream=None,
|
||||
temperature=None,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue