Merge pull request #50360 from BerriAI/litellm_backport_bedrock_converse_service_tier_lts_1_108_x

fix(bedrock): backport Converse service_tier fix to lts/1.108.x

litellm/llms/bedrock/chat/converse_transformation.py

@@ -160,6 +160,10 @@ def _tools_the_model_accepts(
     return [tool_with_sanitized_parameters(tool, drop_lookaround_regex_patterns) for tool in tools]
 
 
+BEDROCK_SERVICE_TIERS: Final = ("default", "flex", "priority", "reserved")
+_BEDROCK_SERVICE_TIER_ALIASES: Final = types.MappingProxyType({"standard_only": "default"})
+
+
 class AmazonConverseConfig(BaseConfig):
     """
     Reference - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html
@@ -1200,8 +1204,8 @@ class AmazonConverseConfig(BaseConfig):
                 self._map_context_management_param(value, optional_params)
             if param == "requestMetadata":
                 self._map_request_metadata_param(value, optional_params)
-            if param == "service_tier" and isinstance(value, str):
-                self._map_service_tier_param(value, optional_params)
+            if param == "service_tier":
+                self._map_service_tier_param(_JSON_VALUE.validate_python(value), optional_params, model, drop_params)
 
             if param == "web_search_options" and isinstance(value, dict):
                 # Note: we use `isinstance(value, dict)` instead of `value and isinstance(value, dict)`
@@ -1254,17 +1258,22 @@ class AmazonConverseConfig(BaseConfig):
         if mapped is not None:
             optional_params["context_management"] = mapped
 
-    def _map_service_tier_param(self, value: str, optional_params: dict) -> None:
-        """Map OpenAI service_tier (string) to Bedrock serviceTier (object).
-
-        OpenAI values: "auto", "default", "flex", "priority"
-        Bedrock values: "default", "flex", "priority" (no "auto")
-        """
-        bedrock_tier = value
-        if value == "auto":
-            bedrock_tier = "default"  # Bedrock doesn't support "auto"
-        if bedrock_tier in ("default", "flex", "priority"):
+    def _map_service_tier_param(self, value: object, optional_params: dict, model: str, drop_params: bool) -> None:
+        if value is None or value == "auto":
+            return
+        bedrock_tier: Final = _BEDROCK_SERVICE_TIER_ALIASES.get(value, value) if isinstance(value, str) else None
+        if bedrock_tier in BEDROCK_SERVICE_TIERS:
             optional_params["serviceTier"] = {"type": bedrock_tier}
+            return
+        if not (litellm.drop_params or drop_params):
+            raise litellm.utils.UnsupportedParamsError(
+                message=(
+                    f"{model} does not support service_tier={value!r} on Bedrock. "
+                    "To drop unsupported params, set `litellm.drop_params = True`."
+                ),
+                status_code=400,
+            )
+        verbose_logger.warning("Dropping service_tier=%r for %s, since Bedrock does not serve it.", value, model)
 
     def _translate_response_format_param(
         self,

litellm/llms/bedrock/files/transformation.py

@@ -942,7 +942,11 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         model), so `input`, `instructions`, `max_output_tokens` and the tool
         params translate identically in batch and real time.
         """
-        return responses_batch_body_to_chat_body(openai_request_body)
+        chat_body: Final = responses_batch_body_to_chat_body(openai_request_body)
+        drop_params: Final = openai_request_body.get("drop_params")
+        if drop_params is None:
+            return chat_body
+        return _frozen_mapping(chain(chat_body.items(), (("drop_params", drop_params),)))
 
     @staticmethod
     def _transform_batch_body_to_chat_body(
@@ -1024,7 +1028,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
                 non_default_params=optional_params,
                 optional_params={},
                 model=model,
-                drop_params=False,
+                drop_params=optional_params.get("drop_params") is True,
             )
             return converse_config.transform_request(
                 model=model,
@@ -1038,7 +1042,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         #     like openai.gpt-oss-*, qwen, deepseek, etc.) ---
         return {
             "messages": messages,
-            **optional_params,
+            **{k: v for k, v in optional_params.items() if k != "drop_params"},
         }
 
     def _resolve_batch_record_model_and_provider(

litellm/proxy/db/db_transaction_queue/redis_update_buffer.py

@@ -13,7 +13,6 @@ from itertools import chain
 from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
 
 from pydantic import TypeAdapter
-
 from redis.exceptions import RedisError
 
 from litellm._internal_context import with_service_target

litellm/types/llms/bedrock.py

@@ -244,7 +244,7 @@ class ConverseTokenUsageBlock(TypedDict, total=False):
 
 
 class ServiceTierBlock(TypedDict):
-    type: Literal["priority", "default", "flex"]
+    type: Literal["priority", "default", "flex", "reserved"]
 
 
 class ConverseResponseBlock(TypedDict, total=False):

litellm/utils.py

@@ -434,11 +434,11 @@ if TYPE_CHECKING:
     )
     from litellm.llms.cohere.common_utils import CohereModelInfo
     from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
+    from litellm.llms.custom_llm import CustomLLM
     from litellm.router_utils.get_retry_from_policy import (
         get_num_retries_from_retry_policy,
         reset_retry_policy,
     )
-    from litellm.llms.custom_llm import CustomLLM
     from litellm.types.llms.anthropic import (
         ANTHROPIC_API_ONLY_HEADERS,
         AnthropicThinkingParam,

tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py

@@ -18,7 +18,6 @@ from litellm.types.utils import (
     ChatCompletionMessageToolCall,
     Delta,
     Function,
-    GenericStreamingChunk,
     ModelResponseStream,
     PromptTokensDetails,
     ServerToolUse,

tests/unit/llms/bedrock/chat/test_service_tier.py

@@ -3,11 +3,13 @@ Tests for Bedrock Converse API serviceTier support.
 """
 
 import json
+from unittest.mock import MagicMock
 
 import pytest
 
-
+import litellm
 from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
+from litellm.llms.custom_httpx.http_handler import HTTPHandler
 from litellm.types.llms.bedrock import ServiceTierBlock
 
 
@@ -201,11 +203,8 @@ def test_map_openai_service_tier_flex():
     assert result["serviceTier"] == {"type": "flex"}
 
 
-def test_map_openai_service_tier_auto_maps_to_default():
-    """Test that OpenAI service_tier='auto' maps to Bedrock serviceTier='default'.
-
-    Bedrock doesn't support 'auto', so we map it to 'default'.
-    """
+def test_map_openai_service_tier_auto_sends_no_tier():
+    """`auto` picks no tier, so Bedrock gets no serviceTier and serves its own default."""
     config = AmazonConverseConfig()
 
     result = config.map_openai_params(
@@ -215,10 +214,123 @@ def test_map_openai_service_tier_auto_maps_to_default():
         drop_params=False,
     )
 
-    assert "serviceTier" in result
+    assert "serviceTier" not in result
+
+
+def test_map_openai_service_tier_none_sends_no_tier():
+    config = AmazonConverseConfig()
+
+    result = config.map_openai_params(
+        non_default_params={"service_tier": None},
+        optional_params={},
+        model="anthropic.claude-3-sonnet-20240229-v1:0",
+        drop_params=False,
+    )
+
+    assert "serviceTier" not in result
+
+
+def test_map_openai_service_tier_reserved_reaches_the_request():
+    config = AmazonConverseConfig()
+    model = "anthropic.claude-3-sonnet-20240229-v1:0"
+
+    optional_params = config.map_openai_params(
+        non_default_params={"service_tier": "reserved"},
+        optional_params={},
+        model=model,
+        drop_params=False,
+    )
+    request = config.transform_request(
+        model=model,
+        messages=[{"role": "user", "content": "Hello!"}],
+        optional_params=optional_params,
+        litellm_params={},
+        headers={},
+    )
+
+    assert request["serviceTier"] == {"type": "reserved"}
+
+
+def test_map_openai_service_tier_standard_only_maps_to_default():
+    """`standard_only` is the Anthropic Messages value that means Bedrock's Standard tier."""
+    config = AmazonConverseConfig()
+
+    result = config.map_openai_params(
+        non_default_params={"service_tier": "standard_only"},
+        optional_params={},
+        model="anthropic.claude-3-sonnet-20240229-v1:0",
+        drop_params=False,
+    )
+
     assert result["serviceTier"] == {"type": "default"}
 
 
+@pytest.mark.parametrize("value", ["ultrafast", "fast", "scale", "", {"type": "priority"}, ["fast"], 1])
+def test_map_openai_service_tier_unmapped_value_answers_400(value):
+    config = AmazonConverseConfig()
+    optional_params = {}
+
+    with pytest.raises(litellm.utils.UnsupportedParamsError) as error:
+        config.map_openai_params(
+            non_default_params={"service_tier": value},
+            optional_params=optional_params,
+            model="anthropic.claude-3-sonnet-20240229-v1:0",
+            drop_params=False,
+        )
+
+    assert error.value.status_code == 400
+    assert f"service_tier={value!r}" in error.value.message
+    assert "drop_params" in error.value.message
+    assert "serviceTier" not in optional_params
+
+
+@pytest.mark.parametrize("value", ["ultrafast", {"type": "priority"}])
+def test_map_openai_service_tier_unmapped_value_drops_under_request_drop_params(value):
+    config = AmazonConverseConfig()
+
+    result = config.map_openai_params(
+        non_default_params={"service_tier": value},
+        optional_params={},
+        model="anthropic.claude-3-sonnet-20240229-v1:0",
+        drop_params=True,
+    )
+
+    assert "serviceTier" not in result
+
+
+def test_map_openai_service_tier_unmapped_value_drops_under_global_drop_params(monkeypatch):
+    monkeypatch.setattr(litellm, "drop_params", True)
+    config = AmazonConverseConfig()
+
+    result = config.map_openai_params(
+        non_default_params={"service_tier": "ultrafast"},
+        optional_params={},
+        model="anthropic.claude-3-sonnet-20240229-v1:0",
+        drop_params=False,
+    )
+
+    assert "serviceTier" not in result
+
+
+@pytest.mark.parametrize("model", ["bedrock/converse/us.openai.gpt-6-luna", "bedrock/converse/us.amazon.nova-2-lite-v1:0"])
+def test_completion_refuses_unmapped_service_tier_before_calling_bedrock(model, monkeypatch):
+    monkeypatch.setattr(litellm, "drop_params", False)
+    client = MagicMock(spec=HTTPHandler)
+
+    with pytest.raises(litellm.BadRequestError) as error:
+        litellm.completion(
+            model=model,
+            messages=[{"role": "user", "content": "Hello!"}],
+            service_tier="ultrafast",
+            aws_region_name="us-east-1",
+            client=client,
+        )
+
+    assert error.value.status_code == 400
+    assert "service_tier='ultrafast'" in error.value.message
+    client.post.assert_not_called()
+
+
 # Tests for service_tier in response
 
 

tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py

@@ -22,7 +22,7 @@ from botocore.credentials import Credentials
 
 import litellm
 from litellm.constants import DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET
-from litellm.llms.bedrock.files.transformation import BedrockJsonlFilesTransformation
+from litellm.llms.bedrock.files.transformation import BedrockFilesConfig, BedrockJsonlFilesTransformation
 from tests.unit.proxy.conftest import httpx_transport
 
 
@@ -101,6 +101,49 @@ class TestBedrockFilesTransformation:
         assert "_litellm_undeclared_sentinel" not in model_input, model_input
         assert result[0]["modelInput"]["max_tokens"] == 10
 
+    @staticmethod
+    def _nova_row(
+        url: str = "/v1/chat/completions", model: str = "us.amazon.nova-pro-v1:0", **body_params: object
+    ) -> dict[str, object]:
+        prompt: Final = {"input": "hi"} if url == "/v1/responses" else {"messages": [{"role": "user", "content": "hi"}]}
+        return {
+            "custom_id": "nova-tier-1",
+            "method": "POST",
+            "url": url,
+            "body": {"model": model, **prompt, **body_params},
+        }
+
+    def test_nova_row_with_null_service_tier_sends_no_tier(self):
+        result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(
+            [self._nova_row(service_tier=None)]
+        )
+
+        assert "serviceTier" not in result[0]["modelInput"]
+
+    @pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"])
+    def test_nova_row_with_unmapped_service_tier_fails_the_row_like_real_time(self, url: str):
+        with pytest.raises(litellm.UnsupportedParamsError, match="ultrafast"):
+            BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(
+                [self._nova_row(url, service_tier="ultrafast")]
+            )
+
+    @pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"])
+    def test_nova_row_level_drop_params_drops_the_unmapped_service_tier(self, url: str):
+        result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(
+            [self._nova_row(url, service_tier="ultrafast", drop_params=True)]
+        )
+
+        assert "serviceTier" not in result[0]["modelInput"]
+        assert "drop_params" not in json.dumps(result[0]["modelInput"])
+
+    @pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"])
+    def test_passthrough_row_keeps_row_level_drop_params_out_of_the_model_input(self, url: str):
+        result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(
+            [self._nova_row(url, "openai.gpt-oss-120b-1:0", drop_params=True)]
+        )
+
+        assert "drop_params" not in result[0]["modelInput"]
+
     def test_nova_text_only_uses_converse_format(self):
         """
         Test that Nova models produce Converse API format in batch modelInput.