diff --git a/litellm/llms/bedrock/chat/chat_completions/transformation.py b/litellm/llms/bedrock/chat/chat_completions/transformation.py index dd2277a6c35..c3e0ae333fe 100644 --- a/litellm/llms/bedrock/chat/chat_completions/transformation.py +++ b/litellm/llms/bedrock/chat/chat_completions/transformation.py @@ -42,6 +42,7 @@ from litellm.llms.bedrock.common_utils import ( bedrock_model_is_openai_gpt, bedrock_rejects_stop_sequences, bedrock_runtime_chat_completions_serves_reasoning_inline, + bedrock_serves_service_tier, split_bedrock_region_path, strip_bedrock_routing_prefix, ) @@ -318,6 +319,13 @@ def with_max_completion_tokens(params: Mapping[str, object]) -> Mapping[str, obj ) +def _unsupported_service_tier(model: str, raw_params: Mapping[str, object]) -> frozenset[str]: + service_tier: Final = raw_params.get("service_tier") + if not isinstance(service_tier, str) or bedrock_serves_service_tier(model, service_tier): + return frozenset() + return frozenset(("service_tier",)) + + class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): def __init__(self, aws_signer: BaseAWSLLM | None = None) -> None: super().__init__() @@ -463,11 +471,22 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): drop_params_gated=True, ) rejected_stop: Final = frozenset(("stop",)) if bedrock_rejects_stop_sequences(model) else frozenset[str]() + unsupported_tier: Final = _unsupported_service_tier(model, raw_params) + if unsupported_tier and not (litellm.drop_params or drop_params): + raise litellm.utils.UnsupportedParamsError( + message=( + f"{model} does not support service_tier={raw_params['service_tier']!r} on Bedrock. " + "To drop unsupported params, set `litellm.drop_params = True`." + ), + status_code=400, + ) return dict( without_refused_reasoning_effort( model, with_max_completion_tokens( - _without_params(mapped, refused_while_reasoning | malformed_effort | rejected_stop) + _without_params( + mapped, refused_while_reasoning | malformed_effort | rejected_stop | unsupported_tier + ) ), ) ) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 3800489ec05..5a6ae980ce2 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -68,6 +68,7 @@ from litellm.llms.bedrock.common_utils import ( bedrock_model_supports_regex_lookaround, bedrock_native_output_format, bedrock_native_output_is_opt_in, + bedrock_serves_service_tier, bedrock_supports_tool_search, deployment_model_info, requests_output_config_format, @@ -1214,7 +1215,7 @@ class AmazonConverseConfig(BaseConfig): if param == "requestMetadata": self._map_request_metadata_param(value, optional_params, drop_params) if param == "service_tier" and isinstance(value, str): - self._map_service_tier_param(value, optional_params) + self._map_service_tier_param(value, optional_params, model=model, drop_params=drop_params) if param == "web_search_options" and isinstance(value, dict): # Note: we use `isinstance(value, dict)` instead of `value and isinstance(value, dict)` @@ -1268,17 +1269,27 @@ class AmazonConverseConfig(BaseConfig): if mapped is not None: optional_params["context_management"] = mapped - def _map_service_tier_param(self, value: str, optional_params: dict) -> None: + def _map_service_tier_param(self, value: str, optional_params: dict, model: str, drop_params: bool) -> None: """Map OpenAI service_tier (string) to Bedrock serviceTier (object). OpenAI values: "auto", "default", "flex", "priority" Bedrock values: "default", "flex", "priority" (no "auto") """ - bedrock_tier = value - if value == "auto": - bedrock_tier = "default" # Bedrock doesn't support "auto" - if bedrock_tier in ("default", "flex", "priority"): - optional_params["serviceTier"] = {"type": bedrock_tier} + bedrock_tier: Final = "default" if value == "auto" else value + if bedrock_tier not in ("default", "flex", "priority"): + return + if not bedrock_serves_service_tier(model, bedrock_tier): + if not (litellm.drop_params or drop_params): + raise litellm.utils.UnsupportedParamsError( + message=( + f"{model} does not support service_tier={value!r} on Bedrock. " + "To drop unsupported params, set `litellm.drop_params = True`." + ), + status_code=400, + ) + verbose_logger.debug("Dropping unsupported `service_tier=%s` for Bedrock model=%s.", value, model) + return + optional_params["serviceTier"] = {"type": bedrock_tier} def _translate_response_format_param( self, diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index d6f5d5b6bd7..a7eb15556cf 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1325,6 +1325,30 @@ def _mantle_api_base_from_env() -> str | None: return next((base[: -len(suffix)] for suffix in _MANTLE_OPENAI_BASE_SUFFIXES if base.endswith(suffix)), base) +def bedrock_serves_service_tier(model: str, service_tier: str) -> bool: + """Whether AWS sells this Bedrock model at the requested ``service_tier``. + + AWS enables priority and flex per model, not per provider (Nova 2, gpt-oss and Kimi serve + them, every Claude and GPT 5.6 Terra answer ``The provided service tier is not supported for + this model``, checked live on 2026-10-27). The price map carries a ``*_cost_per_token_`` + row for exactly the models that serve a tier, so a non-null price is the capability signal (the + proxy normalizes every row to the full ModelInfo shape, so key presence alone means nothing). ``default`` + and ``auto`` are always served, and a model absent from the price map has no signal and is + forwarded as before. Regional CRIS rows (``eu.``, ``apac.``, ``us-gov.``) often omit the tier + prices their base row carries, so the base row is consulted too. + """ + from litellm.litellm_core_utils.llm_cost_calc.utils import get_service_tier_cost_key + + cost_key: Final = get_service_tier_cost_key("input_cost_per_token", service_tier) + if cost_key == "input_cost_per_token": + return True + candidates: Final = (*_bedrock_price_map_entries(model), _model_cost_entry(get_bedrock_base_model(model))) + entries: Final = tuple(entry for entry in candidates if entry is not None) + if not entries: + return True + return any(entry.get(cost_key) is not None for entry in entries) + + DROP_DISABLED_REASONING_EFFORT_WARNING: Final = ( "Dropping %s=%r for Bedrock model=%r (drop_params=True) because the model map marks that level unsupported. " "Send a level the model supports." diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index f4dcff1969e..1298bdb586e 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -3,6 +3,7 @@ import json import logging import re +from typing import Final import httpx import pytest @@ -657,7 +658,7 @@ def _assert_remote_images_inlined(content): def test_transform_request_inlines_remote_image_urls(local_cost_map, monkeypatch): - import litellm.litellm_core_utils.prompt_templates.image_handling as image_handling + from litellm.litellm_core_utils.prompt_templates import image_handling monkeypatch.setattr(image_handling, "convert_url_to_base64", lambda url: f"data:image/png;base64,{url}") body = AmazonBedrockRuntimeChatCompletionsConfig().transform_request( @@ -672,7 +673,7 @@ def test_transform_request_inlines_remote_image_urls(local_cost_map, monkeypatch async def test_async_transform_request_inlines_remote_image_urls(local_cost_map, monkeypatch): - import litellm.litellm_core_utils.prompt_templates.image_handling as image_handling + from litellm.litellm_core_utils.prompt_templates import image_handling async def fake_convert(url, timeout=None): return f"data:image/png;base64,{url}" @@ -1724,3 +1725,53 @@ def test_positional_tool_call_ids_are_minted_for_gpt_too(local_cost_map, fake_aw ids = [tool_call.id for tool_call in response.choices[0].message.tool_calls] assert len(set(ids)) == 2 assert all(re.fullmatch(r"call_[0-9a-f]{32}", tool_call_id) for tool_call_id in ids) + + +# GPT 5.6 Terra on Bedrock answers priority and flex with `'priority' is not supported for 'service_tier' +# on this model`, checked live on 2026-10-27 (LIT-9793). Its price-map row carries null tier prices to match. +def test_native_chat_completions_drops_service_tier_the_model_does_not_sell( + local_cost_map: object, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + config: Final = AmazonBedrockRuntimeChatCompletionsConfig() + + result: Final = config.map_openai_params( + non_default_params={"service_tier": "priority", "max_tokens": 5}, + optional_params={}, + model="us.openai.gpt-5.6-terra", + drop_params=True, + ) + + assert "service_tier" not in result + assert result["max_completion_tokens"] == 5 + + +def test_native_chat_completions_raises_on_service_tier_the_model_does_not_sell( + local_cost_map: object, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + config: Final = AmazonBedrockRuntimeChatCompletionsConfig() + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="service_tier='priority'"): + config.map_openai_params( + non_default_params={"service_tier": "priority"}, + optional_params={}, + model="us.openai.gpt-5.6-terra", + drop_params=False, + ) + + +def test_native_chat_completions_keeps_default_service_tier( + local_cost_map: object, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + config: Final = AmazonBedrockRuntimeChatCompletionsConfig() + + result: Final = config.map_openai_params( + non_default_params={"service_tier": "default"}, + optional_params={}, + model="us.openai.gpt-5.6-terra", + drop_params=False, + ) + + assert result["service_tier"] == "default" diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 524caaba7a1..90fa36b1b48 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -11097,6 +11097,71 @@ def test_a_deployment_with_a_batch_control_plane_endpoint_keeps_it_out_of_chat_r assert "aws_bedrock_control_plane_endpoint" not in route.calls[0].request.content.decode() +# Bedrock tier support is per model. Checked live on 2026-10-27 (LIT-9793): Claude Sonnet 4.5 and +# Haiku 4.5 answer priority and flex with `The provided service tier is not supported for this model.`, +# Nova 2 Lite serves both. The price map mirrors that, Claude rows have null tier prices, Nova 2 has real ones. +def test_converse_service_tier_dropped_on_models_without_that_tier_under_drop_params() -> None: + config: Final = AmazonConverseConfig() + + result: Final = config.map_openai_params( + non_default_params={"service_tier": "priority"}, + optional_params={}, + model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", + drop_params=True, + ) + + assert "serviceTier" not in result + + +def test_converse_service_tier_raises_on_models_without_that_tier_without_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + config: Final = AmazonConverseConfig() + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="service_tier='flex'"): + config.map_openai_params( + non_default_params={"service_tier": "flex"}, + optional_params={}, + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", + drop_params=False, + ) + + +@pytest.mark.parametrize("service_tier", ["priority", "flex"]) +def test_converse_service_tier_forwarded_on_models_that_sell_it( + monkeypatch: pytest.MonkeyPatch, service_tier: str +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + config: Final = AmazonConverseConfig() + + result: Final = config.map_openai_params( + non_default_params={"service_tier": service_tier}, + optional_params={}, + model="us.amazon.nova-2-lite-v1:0", + drop_params=False, + ) + + assert result["serviceTier"] == {"type": service_tier} + + +@pytest.mark.parametrize("service_tier", ["default", "auto"]) +def test_converse_default_and_auto_service_tier_always_forwarded( + monkeypatch: pytest.MonkeyPatch, service_tier: str +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + config: Final = AmazonConverseConfig() + + result: Final = config.map_openai_params( + non_default_params={"service_tier": service_tier}, + optional_params={}, + model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", + drop_params=False, + ) + + assert result["serviceTier"] == {"type": "default"} + + def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: return tuple( r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index 20c0e10cb81..ae698923112 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -1,13 +1,20 @@ - -import asyncio, importlib, litellm, litellm.litellm_core_utils.get_model_cost_map as bedrock_govcloud_model_cost_map, pytest +import asyncio +import importlib +from collections.abc import Iterator from typing import Final +from unittest.mock import Mock, patch +import pytest -from litellm.llms.bedrock.common_utils import( +import litellm +import litellm.litellm_core_utils.get_model_cost_map as bedrock_govcloud_model_cost_map +from litellm import completion +from litellm.llms.bedrock.common_utils import ( AmazonBedrockGlobalConfig, BedrockModelInfo, NativeOutputFallback, bedrock_native_output_is_opt_in, + bedrock_serves_service_tier, bedrock_strict_tools_fallback, deployment_model_info, extract_model_name_from_bedrock_arn, @@ -18,11 +25,8 @@ from litellm.llms.bedrock.common_utils import( strip_bedrock_throughput_suffix, without_bedrock_native_output_opt_in, ) -from collections.abc import Iterator -from litellm import completion from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome -from unittest.mock import Mock, patch # --------------------------------------------------------------------------- # # get_bedrock_response_stream_shape lazy-load tests # @@ -50,9 +54,7 @@ def test_bedrock_response_stream_shape_lazy_loads_once(): import litellm.llms.bedrock.common_utils as mod sentinel = MagicMock() - with patch.object( - mod, "_load_bedrock_response_stream_shape", return_value=sentinel - ) as mock_load: + with patch.object(mod, "_load_bedrock_response_stream_shape", return_value=sentinel) as mock_load: assert mod.get_bedrock_response_stream_shape() is sentinel assert mod.get_bedrock_response_stream_shape() is sentinel mock_load.assert_called_once() @@ -99,9 +101,7 @@ def test_bedrock_response_stream_shape_is_structure_shape(): from litellm.llms.bedrock.common_utils import get_bedrock_response_stream_shape loaded_shape = get_bedrock_response_stream_shape() - assert ( - loaded_shape is not None - ), "get_bedrock_response_stream_shape() is None — botocore may not be installed" + assert loaded_shape is not None, "get_bedrock_response_stream_shape() is None — botocore may not be installed" shape: StructureShape = loaded_shape assert isinstance(shape, StructureShape) assert shape.name == "ResponseStream" @@ -166,9 +166,7 @@ def test_deepseek_cris(): Test that DeepSeek models with cross-region inference prefix use converse route """ bedrock_model_info = BedrockModelInfo - bedrock_route = bedrock_model_info.get_bedrock_route( - model="bedrock/us.deepseek.r1-v1:0" - ) + bedrock_route = bedrock_model_info.get_bedrock_route(model="bedrock/us.deepseek.r1-v1:0") assert bedrock_route == "converse" @@ -234,27 +232,19 @@ def test_govcloud_cross_region_inference_prefix(): bedrock_model_info = BedrockModelInfo # Test us-gov prefix is stripped correctly for Claude models - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.anthropic.claude-haiku-4-5-20251001-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-haiku-4-5-20251001-v1:0") assert base_model == "anthropic.claude-haiku-4-5-20251001-v1:0" # Test us-gov prefix is stripped correctly for different Claude versions - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0") assert base_model == "anthropic.claude-sonnet-4-5-20250929-v1:0" # Test us-gov prefix is stripped correctly for Haiku models - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.anthropic.claude-3-haiku-20240307-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-3-haiku-20240307-v1:0") assert base_model == "anthropic.claude-3-haiku-20240307-v1:0" # Test us-gov prefix is stripped correctly for Meta models - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.meta.llama3-8b-instruct-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.meta.llama3-8b-instruct-v1:0") assert base_model == "meta.llama3-8b-instruct-v1:0" @@ -268,23 +258,14 @@ def test_context_window_suffix_stripped_for_cost_lookup(): """ from litellm.llms.bedrock.common_utils import get_bedrock_base_model - assert ( - get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1[1m]") - == "anthropic.claude-opus-4-6-v1" - ) - assert ( - get_bedrock_base_model("us.anthropic.claude-sonnet-4-6[1m]") - == "anthropic.claude-sonnet-4-6" - ) + assert get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1[1m]") == "anthropic.claude-opus-4-6-v1" + assert get_bedrock_base_model("us.anthropic.claude-sonnet-4-6[1m]") == "anthropic.claude-sonnet-4-6" assert ( get_bedrock_base_model("global.anthropic.claude-opus-4-5-20251101-v1:0[1m]") == "anthropic.claude-opus-4-5-20251101-v1:0" ) # Ensure models without suffix are unaffected - assert ( - get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1") - == "anthropic.claude-opus-4-6-v1" - ) + assert get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1") == "anthropic.claude-opus-4-6-v1" # Ensure :51k throughput suffix still works assert ( get_bedrock_base_model("anthropic.claude-3-5-sonnet-20241022-v2:0:51k") @@ -334,9 +315,7 @@ def test_output_config_effort_normalization_uses_model_info_ceiling(monkeypatch) ("us.anthropic.claude-opus-4-7", "xhigh"), ], ) -def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling( - model, expected_ceiling -): +def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling(model, expected_ceiling): from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap model_info = GetModelCostMap.load_local_model_cost_map()[model] @@ -355,54 +334,24 @@ def test_route_prefix_matched_as_path_segment_not_substring(): or a ``/`` boundary. """ # The bedrock_mantle/ provider prefix must NOT be read as the mantle/ route. - assert ( - BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.5") != "mantle" - ) - assert ( - BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.4") == "converse" - ) - assert ( - BedrockModelInfo._explicit_mantle_route("bedrock_mantle/openai.gpt-5.5") - is False - ) + assert BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.5") != "mantle" + assert BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.4") == "converse" + assert BedrockModelInfo._explicit_mantle_route("bedrock_mantle/openai.gpt-5.5") is False # A genuine mantle route still resolves, via the startswith branch... - assert ( - BedrockModelInfo.get_bedrock_route("mantle/anthropic.claude-mythos-preview") - == "mantle" - ) + assert BedrockModelInfo.get_bedrock_route("mantle/anthropic.claude-mythos-preview") == "mantle" # ...and via the mid-path "/mantle/" branch (after the bedrock/ provider prefix). - assert ( - BedrockModelInfo.get_bedrock_route( - "bedrock/mantle/anthropic.claude-mythos-preview" - ) - == "mantle" - ) + assert BedrockModelInfo.get_bedrock_route("bedrock/mantle/anthropic.claude-mythos-preview") == "mantle" def test_model_has_route_prefix_exercises_both_branches(): """``_model_has_route_prefix`` matches on ``startswith`` or a ``/`` boundary only.""" # startswith branch - assert ( - BedrockModelInfo._model_has_route_prefix( - "mantle/anthropic.claude-mythos-preview", "mantle/" - ) - is True - ) + assert BedrockModelInfo._model_has_route_prefix("mantle/anthropic.claude-mythos-preview", "mantle/") is True # f"/{prefix}" boundary branch - assert ( - BedrockModelInfo._model_has_route_prefix( - "bedrock/mantle/anthropic.claude-mythos-preview", "mantle/" - ) - is True - ) + assert BedrockModelInfo._model_has_route_prefix("bedrock/mantle/anthropic.claude-mythos-preview", "mantle/") is True # neither branch: the token only appears glued to another segment - assert ( - BedrockModelInfo._model_has_route_prefix( - "bedrock_mantle/openai.gpt-5.5", "mantle/" - ) - is False - ) + assert BedrockModelInfo._model_has_route_prefix("bedrock_mantle/openai.gpt-5.5", "mantle/") is False @pytest.mark.parametrize( @@ -452,16 +401,10 @@ def test_explicit_invoke_route_does_not_match_async_invoke(): """ async_invoke_model = "async_invoke/twelvelabs.marengo-embed-2-7-v1:0" assert BedrockModelInfo._explicit_invoke_route(async_invoke_model) is False - assert ( - BedrockModelInfo._explicit_invoke_route(f"bedrock/{async_invoke_model}") - is False - ) + assert BedrockModelInfo._explicit_invoke_route(f"bedrock/{async_invoke_model}") is False # ...while async_invoke/ is still detected as its own route. assert BedrockModelInfo._explicit_async_invoke_route(async_invoke_model) is True - assert ( - BedrockModelInfo._explicit_async_invoke_route(f"bedrock/{async_invoke_model}") - is True - ) + assert BedrockModelInfo._explicit_async_invoke_route(f"bedrock/{async_invoke_model}") is True def test_capability_lookups_fall_back_to_base_model_when_regional_entry_lacks_field(monkeypatch): @@ -1285,6 +1228,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="session") def event_loop(): try: @@ -1294,6 +1238,7 @@ def event_loop(): yield loop loop.close() + @pytest.fixture(scope="function") def setup_and_teardown(event_loop): import litellm @@ -1327,6 +1272,7 @@ def setup_and_teardown(event_loop): if pending: event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "set_verbose": getattr(litellm, "set_verbose", False), @@ -1341,6 +1287,7 @@ _SCALAR_DEFAULTS = { "cohere_key": getattr(litellm, "cohere_key", None), } + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestStripBedrockRoutingPrefix: """Tests for strip_bedrock_routing_prefix function.""" @@ -1371,6 +1318,7 @@ class TestStripBedrockRoutingPrefix: == "anthropic.claude-3-sonnet-20240229-v1:0" ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestStripBedrockThroughputSuffix: """Tests for strip_bedrock_throughput_suffix function.""" @@ -1398,6 +1346,7 @@ class TestStripBedrockThroughputSuffix: def test_strip_throughput_suffix(self, input_model, expected): assert strip_bedrock_throughput_suffix(input_model) == expected + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestExtractModelNameFromBedrockArn: """Tests for extract_model_name_from_bedrock_arn function.""" @@ -1418,6 +1367,7 @@ class TestExtractModelNameFromBedrockArn: arn = "ARN:aws:bedrock:us-east-1:123456789012:model/my-model" assert extract_model_name_from_bedrock_arn(arn) == "my-model" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestGetBedrockCrossRegionInferenceRegions: """Tests for get_bedrock_cross_region_inference_regions function.""" @@ -1433,6 +1383,7 @@ class TestGetBedrockCrossRegionInferenceRegions: regions = get_bedrock_cross_region_inference_regions() assert isinstance(regions, list) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestGetBedrockBaseModel: """Tests for get_bedrock_base_model function.""" @@ -1496,6 +1447,7 @@ class TestGetBedrockBaseModel: """Test that throughput tier suffixes like :51k are stripped. Issue #19113.""" assert get_bedrock_base_model(input_model) == expected + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestBedrockModelInfoWrappers: """Tests that BedrockModelInfo methods correctly wrap standalone functions.""" @@ -1517,6 +1469,7 @@ class TestBedrockModelInfoWrappers: model = "bedrock/converse/claude-3" assert BedrockModelInfo.get_non_litellm_routing_model_name(model) == strip_bedrock_routing_prefix(model) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestBedrockTokenCounter: """Tests for BedrockTokenCounter class.""" @@ -1553,12 +1506,14 @@ class TestBedrockTokenCounter: ) assert result is None + @pytest.fixture def _pr4_bedrock_env(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("AWS_ACCESS_KEY_ID", "pr4-test-aws-access-key") monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "pr4-test-aws-secret-key") monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + @pytest.fixture(scope="module") def _use_local_model_cost_map() -> Iterator[None]: with pytest.MonkeyPatch.context() as monkeypatch: @@ -1567,6 +1522,7 @@ def _use_local_model_cost_map() -> Iterator[None]: importlib.reload(litellm) yield + @pytest.mark.usefixtures( "_pr4_bedrock_env", "_use_local_model_cost_map", @@ -2107,6 +2063,7 @@ class TestBedrockGovCloudSupport: @pytest.mark.parametrize("missing", ["botocore", "unrelated_dependency"]) def test_event_decoder_reports_only_missing_aws_dependency(missing): from unittest.mock import patch + from litellm.llms.bedrock.common_utils import BedrockEventStreamDecoderBase failure = ModuleNotFoundError("missing dependency", name=missing) @@ -2143,8 +2100,12 @@ def test_bedrock_capability_model_reads_base_model_only_for_application_profile_ assert bedrock_capability_model(model, base_model) == expected -_FORMAT_REJECTION: Final = "The model returned the following errors: output_config.format: Extra inputs are not permitted" -_COMPILE_REJECTION: Final = "The model returned the following errors: compilation exceeded limits: Schema is too complex." +_FORMAT_REJECTION: Final = ( + "The model returned the following errors: output_config.format: Extra inputs are not permitted" +) +_COMPILE_REJECTION: Final = ( + "The model returned the following errors: compilation exceeded limits: Schema is too complex." +) _SCHEMA_PATTERN_REJECTION: Final = ( "The model returned the following errors: output_config.format.schema: Unsupported regex feature in pattern " "field: pattern is too complex for structured output: reduce the {n,m} upper bound, narrow the character class " @@ -2245,7 +2206,10 @@ def test_bedrock_strict_tools_fallback_drops_only_strict_and_keeps_everything_el assert fallback == NativeOutputFallback( optional_params={ "tools": [ - {**strict_tool, "function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}}}, + { + **strict_tool, + "function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}}, + }, plain_tool, anthropic_tool, {"toolSpec": {"name": "spec", "inputSchema": {"json": {"type": "object"}}}}, @@ -2300,3 +2264,45 @@ def test_bedrock_strict_tools_fallback_retries_only_when_the_request_sent_strict fallback: Final = bedrock_strict_tools_fallback(model, {"tools": tools}, litellm_params) assert (fallback is not None) == sent_strict + + +_TIER_PRICED_ROW: Final = { + "litellm_provider": "bedrock_converse", + "input_cost_per_token": 1e-6, + "input_cost_per_token_priority": 2e-6, + "input_cost_per_token_flex": 5e-7, +} +_STANDARD_ONLY_ROW: Final = { + "litellm_provider": "bedrock_converse", + "input_cost_per_token": 1e-6, + "input_cost_per_token_priority": None, + "input_cost_per_token_flex": None, +} + + +@pytest.mark.parametrize( + ("model", "service_tier", "expected"), + [ + ("tiered.model-v1:0", "priority", True), + ("tiered.model-v1:0", "flex", True), + ("bedrock/us-west-2/tiered.model-v1:0", "flex", True), + ("eu.tiered.model-v1:0", "priority", True), + ("standard.model-v1:0", "priority", False), + ("standard.model-v1:0", "flex", False), + ("standard.model-v1:0", "default", True), + ("standard.model-v1:0", "auto", True), + ("a-model-the-price-map-has-never-heard-of", "priority", True), + ], +) +def test_bedrock_serves_service_tier_follows_price_map_tier_rows( + monkeypatch: pytest.MonkeyPatch, model: str, service_tier: str, expected: bool +) -> None: + """A Bedrock model serves a tier exactly when its price-map row carries a non-null price for that tier. + The proxy expands every row to the full ModelInfo shape, so a null price (standard-only row) is no signal, + `default` and `auto` are always served, and an unknown model keeps being forwarded. A regional CRIS row + that omits the tier prices does not hide the base row that carries them.""" + monkeypatch.setitem(litellm.model_cost, "tiered.model-v1:0", dict(_TIER_PRICED_ROW)) + monkeypatch.setitem(litellm.model_cost, "eu.tiered.model-v1:0", dict(_STANDARD_ONLY_ROW)) + monkeypatch.setitem(litellm.model_cost, "standard.model-v1:0", dict(_STANDARD_ONLY_ROW)) + + assert bedrock_serves_service_tier(model, service_tier) is expected