diff --git a/litellm/llms/bedrock/chat/chat_completions/transformation.py b/litellm/llms/bedrock/chat/chat_completions/transformation.py index 123e4a9e6b4..131d4afae5b 100644 --- a/litellm/llms/bedrock/chat/chat_completions/transformation.py +++ b/litellm/llms/bedrock/chat/chat_completions/transformation.py @@ -41,6 +41,7 @@ from litellm.llms.bedrock.common_utils import ( bedrock_model_is_openai_gpt, bedrock_rejects_stop_sequences, bedrock_runtime_chat_completions_serves_reasoning_inline, + bedrock_serves_service_tier, split_bedrock_region_path, ) from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler @@ -309,6 +310,13 @@ def with_max_completion_tokens(params: Mapping[str, object]) -> Mapping[str, obj ) +def _unsupported_service_tier(model: str, raw_params: Mapping[str, object]) -> frozenset[str]: + service_tier: Final = raw_params.get("service_tier") + if not isinstance(service_tier, str) or bedrock_serves_service_tier(model, service_tier): + return frozenset() + return frozenset(("service_tier",)) + + class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): def __init__(self, aws_signer: BaseAWSLLM | None = None) -> None: super().__init__() @@ -446,11 +454,22 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): status_code=400, ) rejected_stop: Final = frozenset(("stop",)) if bedrock_rejects_stop_sequences(model) else frozenset[str]() + unsupported_tier: Final = _unsupported_service_tier(model, raw_params) + if unsupported_tier and not (litellm.drop_params or drop_params): + raise litellm.utils.UnsupportedParamsError( + message=( + f"{model} does not support service_tier={raw_params['service_tier']!r} on Bedrock. " + "To drop unsupported params, set `litellm.drop_params = True`." + ), + status_code=400, + ) return dict( without_refused_reasoning_effort( model, with_max_completion_tokens( - _without_params(mapped, refused_while_reasoning | malformed_effort | rejected_stop) + _without_params( + mapped, refused_while_reasoning | malformed_effort | rejected_stop | unsupported_tier + ) ), ) ) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index f54c352fd6e..9e6352c3c2f 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -64,6 +64,7 @@ from litellm.llms.bedrock.common_utils import ( bedrock_model_supports_regex_lookaround, bedrock_native_output_format, bedrock_native_output_is_opt_in, + bedrock_serves_service_tier, bedrock_supports_tool_search, deployment_model_info, requests_output_config_format, @@ -1210,7 +1211,7 @@ class AmazonConverseConfig(BaseConfig): if param == "requestMetadata": self._map_request_metadata_param(value, optional_params, drop_params) if param == "service_tier" and isinstance(value, str): - self._map_service_tier_param(value, optional_params) + self._map_service_tier_param(value, optional_params, model=model, drop_params=drop_params) if param == "web_search_options" and isinstance(value, dict): # Note: we use `isinstance(value, dict)` instead of `value and isinstance(value, dict)` @@ -1264,17 +1265,27 @@ class AmazonConverseConfig(BaseConfig): if mapped is not None: optional_params["context_management"] = mapped - def _map_service_tier_param(self, value: str, optional_params: dict) -> None: + def _map_service_tier_param(self, value: str, optional_params: dict, model: str, drop_params: bool) -> None: """Map OpenAI service_tier (string) to Bedrock serviceTier (object). OpenAI values: "auto", "default", "flex", "priority" Bedrock values: "default", "flex", "priority" (no "auto") """ - bedrock_tier = value - if value == "auto": - bedrock_tier = "default" # Bedrock doesn't support "auto" - if bedrock_tier in ("default", "flex", "priority"): - optional_params["serviceTier"] = {"type": bedrock_tier} + bedrock_tier: Final = "default" if value == "auto" else value + if bedrock_tier not in ("default", "flex", "priority"): + return + if not bedrock_serves_service_tier(model, bedrock_tier): + if not (litellm.drop_params or drop_params): + raise litellm.utils.UnsupportedParamsError( + message=( + f"{model} does not support service_tier={value!r} on Bedrock. " + "To drop unsupported params, set `litellm.drop_params = True`." + ), + status_code=400, + ) + verbose_logger.debug("Dropping unsupported `service_tier=%s` for Bedrock model=%s.", value, model) + return + optional_params["serviceTier"] = {"type": bedrock_tier} def _translate_response_format_param( self, diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index fde995c7907..2ca7bef7e0a 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1314,6 +1314,28 @@ def _mantle_api_base_from_env() -> str | None: return next((base[: -len(suffix)] for suffix in _MANTLE_OPENAI_BASE_SUFFIXES if base.endswith(suffix)), base) +def bedrock_serves_service_tier(model: str, service_tier: str) -> bool: + """Whether AWS sells this Bedrock model at the requested ``service_tier``. + + AWS enables priority and flex per model, not per provider (Nova 2, gpt-oss and Kimi serve + them, every Claude and GPT 5.6 Terra answer ``The provided service tier is not supported for + this model``, checked live on 2026-10-27). The price map carries a ``*_cost_per_token_`` + row for exactly the models that serve a tier, so a non-null price is the capability signal (the + proxy normalizes every row to the full ModelInfo shape, so key presence alone means nothing). ``default`` + and ``auto`` are always served, and a model absent from the price map has no signal and is + forwarded as before. + """ + from litellm.litellm_core_utils.llm_cost_calc.utils import get_service_tier_cost_key + + cost_key: Final = get_service_tier_cost_key("input_cost_per_token", service_tier) + if cost_key == "input_cost_per_token": + return True + entries: Final = tuple(entry for entry in _bedrock_price_map_entries(model) if entry is not None) + if not entries: + return True + return any(entry.get(cost_key) is not None for entry in entries) + + def bedrock_reasoning_effort_disabled(model: str, effort: str) -> bool: from litellm.utils import is_explicitly_disabled_factory diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 3faae2e3289..cf65facf6ee 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -1704,3 +1704,47 @@ def test_positional_tool_call_ids_are_minted_for_gpt_too(local_cost_map, fake_aw ids = [tool_call.id for tool_call in response.choices[0].message.tool_calls] assert len(set(ids)) == 2 assert all(re.fullmatch(r"call_[0-9a-f]{32}", tool_call_id) for tool_call_id in ids) + + +def test_native_chat_completions_drops_service_tier_the_model_does_not_sell(local_cost_map, monkeypatch): + """GPT 5.6 Terra on Bedrock answers priority/flex with `'priority' is not supported for + 'service_tier' on this model`, so under drop_params the tier is dropped before the call.""" + monkeypatch.setattr(litellm, "drop_params", False) + config = AmazonBedrockRuntimeChatCompletionsConfig() + + result = config.map_openai_params( + non_default_params={"service_tier": "priority", "max_tokens": 5}, + optional_params={}, + model="us.openai.gpt-5.6-terra", + drop_params=True, + ) + + assert "service_tier" not in result + assert result["max_completion_tokens"] == 5 + + +def test_native_chat_completions_raises_on_service_tier_the_model_does_not_sell(local_cost_map, monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + config = AmazonBedrockRuntimeChatCompletionsConfig() + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="service_tier='priority'"): + config.map_openai_params( + non_default_params={"service_tier": "priority"}, + optional_params={}, + model="us.openai.gpt-5.6-terra", + drop_params=False, + ) + + +def test_native_chat_completions_keeps_default_service_tier(local_cost_map, monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + config = AmazonBedrockRuntimeChatCompletionsConfig() + + result = config.map_openai_params( + non_default_params={"service_tier": "default"}, + optional_params={}, + model="us.openai.gpt-5.6-terra", + drop_params=False, + ) + + assert result["service_tier"] == "default" diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 154797bab5a..c8cf0a0b304 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -10789,3 +10789,61 @@ def test_a_deployment_with_a_batch_control_plane_endpoint_keeps_it_out_of_chat_r assert response.choices[0].message.content == "ok" assert "aws_bedrock_control_plane_endpoint" not in route.calls[0].request.content.decode() + + +def test_converse_service_tier_dropped_on_models_without_that_tier_under_drop_params(): + """Bedrock sells priority/flex per model. Claude has neither, so with drop_params the tier must + be dropped instead of forwarded to AWS's `The provided service tier is not supported` 400.""" + config = AmazonConverseConfig() + + result = config.map_openai_params( + non_default_params={"service_tier": "priority"}, + optional_params={}, + model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", + drop_params=True, + ) + + assert "serviceTier" not in result + + +def test_converse_service_tier_raises_on_models_without_that_tier_without_drop_params(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + config = AmazonConverseConfig() + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="service_tier='flex'"): + config.map_openai_params( + non_default_params={"service_tier": "flex"}, + optional_params={}, + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", + drop_params=False, + ) + + +@pytest.mark.parametrize("service_tier", ["priority", "flex"]) +def test_converse_service_tier_forwarded_on_models_that_sell_it(monkeypatch, service_tier): + monkeypatch.setattr(litellm, "drop_params", False) + config = AmazonConverseConfig() + + result = config.map_openai_params( + non_default_params={"service_tier": service_tier}, + optional_params={}, + model="us.amazon.nova-2-lite-v1:0", + drop_params=False, + ) + + assert result["serviceTier"] == {"type": service_tier} + + +@pytest.mark.parametrize("service_tier", ["default", "auto"]) +def test_converse_default_and_auto_service_tier_always_forwarded(monkeypatch, service_tier): + monkeypatch.setattr(litellm, "drop_params", False) + config = AmazonConverseConfig() + + result = config.map_openai_params( + non_default_params={"service_tier": service_tier}, + optional_params={}, + model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", + drop_params=False, + ) + + assert result["serviceTier"] == {"type": "default"} diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index dd89d9c46b6..e02ff1c8d26 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -1,13 +1,21 @@ -import asyncio, importlib, litellm, litellm.litellm_core_utils.get_model_cost_map as bedrock_govcloud_model_cost_map, pytest +import asyncio +import importlib +from collections.abc import Iterator from typing import Final +from unittest.mock import Mock, patch +import pytest -from litellm.llms.bedrock.common_utils import( +import litellm +import litellm.litellm_core_utils.get_model_cost_map as bedrock_govcloud_model_cost_map +from litellm import completion +from litellm.llms.bedrock.common_utils import ( AmazonBedrockGlobalConfig, BedrockModelInfo, NativeOutputFallback, bedrock_native_output_is_opt_in, + bedrock_serves_service_tier, bedrock_strict_tools_fallback, deployment_model_info, extract_model_name_from_bedrock_arn, @@ -18,11 +26,8 @@ from litellm.llms.bedrock.common_utils import( strip_bedrock_throughput_suffix, without_bedrock_native_output_opt_in, ) -from collections.abc import Iterator -from litellm import completion from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome -from unittest.mock import Mock, patch # --------------------------------------------------------------------------- # # get_bedrock_response_stream_shape lazy-load tests # @@ -2036,6 +2041,7 @@ class TestBedrockGovCloudSupport: @pytest.mark.parametrize("missing", ["botocore", "unrelated_dependency"]) def test_event_decoder_reports_only_missing_aws_dependency(missing): from unittest.mock import patch + from litellm.llms.bedrock.common_utils import BedrockEventStreamDecoderBase failure = ModuleNotFoundError("missing dependency", name=missing) @@ -2229,3 +2235,38 @@ def test_bedrock_strict_tools_fallback_retries_only_when_the_request_sent_strict fallback: Final = bedrock_strict_tools_fallback(model, {"tools": tools}, litellm_params) assert (fallback is not None) == sent_strict + + +@pytest.mark.parametrize( + ("model", "service_tier", "expected"), + [ + ("us.anthropic.claude-sonnet-4-5-20250929-v1:0", "priority", False), + ("us.anthropic.claude-sonnet-4-5-20250929-v1:0", "default", True), + ("us.anthropic.claude-sonnet-4-5-20250929-v1:0", "auto", True), + ("us.amazon.nova-2-lite-v1:0", "flex", True), + ("bedrock/us-west-2/moonshotai.kimi-k2.5", "flex", True), + ("a-model-the-price-map-has-never-heard-of", "priority", True), + ], +) +def test_bedrock_serves_service_tier_follows_price_map_tier_rows(model, service_tier, expected): + """A Bedrock model serves a tier exactly when its price-map row carries that tier's pricing. + Unknown models have no signal and keep being forwarded.""" + assert bedrock_serves_service_tier(model, service_tier) is expected + + +def test_bedrock_serves_service_tier_ignores_null_tier_prices_from_normalized_rows(monkeypatch): + """The proxy expands every price-map row to the full ModelInfo shape, so a Claude row carries + `input_cost_per_token_priority: None`. A null price is no signal that AWS sells the tier.""" + monkeypatch.setitem( + litellm.model_cost, + "normalized.claude-row", + {"litellm_provider": "bedrock_converse", "input_cost_per_token": 1e-6, "input_cost_per_token_priority": None}, + ) + monkeypatch.setitem( + litellm.model_cost, + "normalized.nova-row", + {"litellm_provider": "bedrock_converse", "input_cost_per_token": 1e-6, "input_cost_per_token_priority": 2e-6}, + ) + + assert bedrock_serves_service_tier("normalized.claude-row", "priority") is False + assert bedrock_serves_service_tier("normalized.nova-row", "priority") is True