From 8a60580280eb41fae3e6461b05a3c5c0fd44a4ce Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:57:46 -0800 Subject: [PATCH 1/3] fix(bedrock): answer 400 for service tiers Bedrock cannot serve, forward reserved Converse used to drop any service_tier outside default, flex, and priority with no error, rewrite auto to default, and drop reserved. Now reserved and standard_only map to Bedrock tiers, auto sends no tier, and any other value answers 400 unless drop_params is on. Nova batch rows honor the row's drop_params. Backport to lts/1.108.x of ce94d2894d5 (litellm_bedrock_converse_service_tier_values), whose main PR is pending. Adapted to the line: lts has no REQUEST_PARAMS, so the call passes the value through TypeAdapter(object) and keeps the line's two-argument requestMetadata call. --- .../bedrock/chat/converse_transformation.py | 33 +++-- litellm/llms/bedrock/files/transformation.py | 2 +- litellm/types/llms/bedrock.py | 2 +- .../llms/bedrock/chat/test_service_tier.py | 126 +++++++++++++++++- .../test_bedrock_files_transformation.py | 35 ++++- 5 files changed, 176 insertions(+), 22 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 6984fc1c1cd..4f4ba66088e 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -160,6 +160,10 @@ def _tools_the_model_accepts( return [tool_with_sanitized_parameters(tool, drop_lookaround_regex_patterns) for tool in tools] +BEDROCK_SERVICE_TIERS: Final = ("default", "flex", "priority", "reserved") +_BEDROCK_SERVICE_TIER_ALIASES: Final = types.MappingProxyType({"standard_only": "default"}) + + class AmazonConverseConfig(BaseConfig): """ Reference - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html @@ -1200,8 +1204,8 @@ class AmazonConverseConfig(BaseConfig): self._map_context_management_param(value, optional_params) if param == "requestMetadata": self._map_request_metadata_param(value, optional_params) - if param == "service_tier" and isinstance(value, str): - self._map_service_tier_param(value, optional_params) + if param == "service_tier": + self._map_service_tier_param(_JSON_VALUE.validate_python(value), optional_params, model, drop_params) if param == "web_search_options" and isinstance(value, dict): # Note: we use `isinstance(value, dict)` instead of `value and isinstance(value, dict)` @@ -1254,17 +1258,22 @@ class AmazonConverseConfig(BaseConfig): if mapped is not None: optional_params["context_management"] = mapped - def _map_service_tier_param(self, value: str, optional_params: dict) -> None: - """Map OpenAI service_tier (string) to Bedrock serviceTier (object). - - OpenAI values: "auto", "default", "flex", "priority" - Bedrock values: "default", "flex", "priority" (no "auto") - """ - bedrock_tier = value - if value == "auto": - bedrock_tier = "default" # Bedrock doesn't support "auto" - if bedrock_tier in ("default", "flex", "priority"): + def _map_service_tier_param(self, value: object, optional_params: dict, model: str, drop_params: bool) -> None: + if value is None or value == "auto": + return + bedrock_tier: Final = _BEDROCK_SERVICE_TIER_ALIASES.get(value, value) if isinstance(value, str) else None + if bedrock_tier in BEDROCK_SERVICE_TIERS: optional_params["serviceTier"] = {"type": bedrock_tier} + return + if not (litellm.drop_params or drop_params): + raise litellm.utils.UnsupportedParamsError( + message=( + f"{model} does not support service_tier={value!r} on Bedrock. " + "To drop unsupported params, set `litellm.drop_params = True`." + ), + status_code=400, + ) + verbose_logger.warning("Dropping service_tier=%r for %s, since Bedrock does not serve it.", value, model) def _translate_response_format_param( self, diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 4d40e0dec85..7d3ac50faa0 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -1024,7 +1024,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): non_default_params=optional_params, optional_params={}, model=model, - drop_params=False, + drop_params=optional_params.get("drop_params") is True, ) return converse_config.transform_request( model=model, diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index ff4f2c4fd1e..e7bef9b3946 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -244,7 +244,7 @@ class ConverseTokenUsageBlock(TypedDict, total=False): class ServiceTierBlock(TypedDict): - type: Literal["priority", "default", "flex"] + type: Literal["priority", "default", "flex", "reserved"] class ConverseResponseBlock(TypedDict, total=False): diff --git a/tests/unit/llms/bedrock/chat/test_service_tier.py b/tests/unit/llms/bedrock/chat/test_service_tier.py index ce9dc4d745e..e248a64fba4 100644 --- a/tests/unit/llms/bedrock/chat/test_service_tier.py +++ b/tests/unit/llms/bedrock/chat/test_service_tier.py @@ -3,11 +3,13 @@ Tests for Bedrock Converse API serviceTier support. """ import json +from unittest.mock import MagicMock import pytest - +import litellm from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.types.llms.bedrock import ServiceTierBlock @@ -201,11 +203,8 @@ def test_map_openai_service_tier_flex(): assert result["serviceTier"] == {"type": "flex"} -def test_map_openai_service_tier_auto_maps_to_default(): - """Test that OpenAI service_tier='auto' maps to Bedrock serviceTier='default'. - - Bedrock doesn't support 'auto', so we map it to 'default'. - """ +def test_map_openai_service_tier_auto_sends_no_tier(): + """`auto` picks no tier, so Bedrock gets no serviceTier and serves its own default.""" config = AmazonConverseConfig() result = config.map_openai_params( @@ -215,10 +214,123 @@ def test_map_openai_service_tier_auto_maps_to_default(): drop_params=False, ) - assert "serviceTier" in result + assert "serviceTier" not in result + + +def test_map_openai_service_tier_none_sends_no_tier(): + config = AmazonConverseConfig() + + result = config.map_openai_params( + non_default_params={"service_tier": None}, + optional_params={}, + model="anthropic.claude-3-sonnet-20240229-v1:0", + drop_params=False, + ) + + assert "serviceTier" not in result + + +def test_map_openai_service_tier_reserved_reaches_the_request(): + config = AmazonConverseConfig() + model = "anthropic.claude-3-sonnet-20240229-v1:0" + + optional_params = config.map_openai_params( + non_default_params={"service_tier": "reserved"}, + optional_params={}, + model=model, + drop_params=False, + ) + request = config.transform_request( + model=model, + messages=[{"role": "user", "content": "Hello!"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert request["serviceTier"] == {"type": "reserved"} + + +def test_map_openai_service_tier_standard_only_maps_to_default(): + """`standard_only` is the Anthropic Messages value that means Bedrock's Standard tier.""" + config = AmazonConverseConfig() + + result = config.map_openai_params( + non_default_params={"service_tier": "standard_only"}, + optional_params={}, + model="anthropic.claude-3-sonnet-20240229-v1:0", + drop_params=False, + ) + assert result["serviceTier"] == {"type": "default"} +@pytest.mark.parametrize("value", ["ultrafast", "fast", "scale", "", {"type": "priority"}, ["fast"], 1]) +def test_map_openai_service_tier_unmapped_value_answers_400(value): + config = AmazonConverseConfig() + optional_params = {} + + with pytest.raises(litellm.utils.UnsupportedParamsError) as error: + config.map_openai_params( + non_default_params={"service_tier": value}, + optional_params=optional_params, + model="anthropic.claude-3-sonnet-20240229-v1:0", + drop_params=False, + ) + + assert error.value.status_code == 400 + assert f"service_tier={value!r}" in error.value.message + assert "drop_params" in error.value.message + assert "serviceTier" not in optional_params + + +@pytest.mark.parametrize("value", ["ultrafast", {"type": "priority"}]) +def test_map_openai_service_tier_unmapped_value_drops_under_request_drop_params(value): + config = AmazonConverseConfig() + + result = config.map_openai_params( + non_default_params={"service_tier": value}, + optional_params={}, + model="anthropic.claude-3-sonnet-20240229-v1:0", + drop_params=True, + ) + + assert "serviceTier" not in result + + +def test_map_openai_service_tier_unmapped_value_drops_under_global_drop_params(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", True) + config = AmazonConverseConfig() + + result = config.map_openai_params( + non_default_params={"service_tier": "ultrafast"}, + optional_params={}, + model="anthropic.claude-3-sonnet-20240229-v1:0", + drop_params=False, + ) + + assert "serviceTier" not in result + + +@pytest.mark.parametrize("model", ["bedrock/converse/us.openai.gpt-6-luna", "bedrock/converse/us.amazon.nova-2-lite-v1:0"]) +def test_completion_refuses_unmapped_service_tier_before_calling_bedrock(model, monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + client = MagicMock(spec=HTTPHandler) + + with pytest.raises(litellm.BadRequestError) as error: + litellm.completion( + model=model, + messages=[{"role": "user", "content": "Hello!"}], + service_tier="ultrafast", + aws_region_name="us-east-1", + client=client, + ) + + assert error.value.status_code == 400 + assert "service_tier='ultrafast'" in error.value.message + client.post.assert_not_called() + + # Tests for service_tier in response diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index 425beefac8a..5dc242386d3 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -22,7 +22,7 @@ from botocore.credentials import Credentials import litellm from litellm.constants import DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET -from litellm.llms.bedrock.files.transformation import BedrockJsonlFilesTransformation +from litellm.llms.bedrock.files.transformation import BedrockFilesConfig, BedrockJsonlFilesTransformation from tests.unit.proxy.conftest import httpx_transport @@ -101,6 +101,39 @@ class TestBedrockFilesTransformation: assert "_litellm_undeclared_sentinel" not in model_input, model_input assert result[0]["modelInput"]["max_tokens"] == 10 + @staticmethod + def _nova_row(**body_params: object) -> dict[str, object]: + return { + "custom_id": "nova-tier-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "us.amazon.nova-pro-v1:0", + "messages": [{"role": "user", "content": "hi"}], + **body_params, + }, + } + + def test_nova_row_with_null_service_tier_sends_no_tier(self): + result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( + [self._nova_row(service_tier=None)] + ) + + assert "serviceTier" not in result[0]["modelInput"] + + def test_nova_row_with_unmapped_service_tier_fails_the_row_like_real_time(self): + with pytest.raises(litellm.UnsupportedParamsError, match="ultrafast"): + BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( + [self._nova_row(service_tier="ultrafast")] + ) + + def test_nova_row_level_drop_params_drops_the_unmapped_service_tier(self): + result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( + [self._nova_row(service_tier="ultrafast", drop_params=True)] + ) + + assert "serviceTier" not in result[0]["modelInput"] + def test_nova_text_only_uses_converse_format(self): """ Test that Nova models produce Converse API format in batch modelInput. From f2cd45a8abe5dcfd318912b28ab83c80b8f981f9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:02:55 -0800 Subject: [PATCH 2/3] style: clear the ruff reds lts/1.108.x already has Sorts two imports ruff flags (I001) in redis_update_buffer.py and utils.py, the same hunks #50348 carries, and drops a duplicate GenericStreamingChunk import (F811) in a streaming test. No behavior change. --- litellm/proxy/db/db_transaction_queue/redis_update_buffer.py | 1 - litellm/utils.py | 2 +- .../litellm_core_utils/test_streaming_chunk_builder_utils.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py index 22f55ea9153..8fbaa7e3a9a 100644 --- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py +++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py @@ -13,7 +13,6 @@ from itertools import chain from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast from pydantic import TypeAdapter - from redis.exceptions import RedisError from litellm._internal_context import with_service_target diff --git a/litellm/utils.py b/litellm/utils.py index 51dc6db456a..71c9ca0a16e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -434,11 +434,11 @@ if TYPE_CHECKING: ) from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + from litellm.llms.custom_llm import CustomLLM from litellm.router_utils.get_retry_from_policy import ( get_num_retries_from_retry_policy, reset_retry_policy, ) - from litellm.llms.custom_llm import CustomLLM from litellm.types.llms.anthropic import ( ANTHROPIC_API_ONLY_HEADERS, AnthropicThinkingParam, diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index a46c8bed4ad..716c0af236c 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -18,7 +18,6 @@ from litellm.types.utils import ( ChatCompletionMessageToolCall, Delta, Function, - GenericStreamingChunk, ModelResponseStream, PromptTokensDetails, ServerToolUse, From a97503a3930e9f36261fc76ab3622f05b9856f2d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 01:37:23 -0800 Subject: [PATCH 3/3] fix(bedrock): keep a responses batch row's drop_params A /v1/responses batch row went through the Responses-to-Chat bridge, which keeps only Responses fields, so the row's own drop_params was lost while its service_tier still reached the Nova mapper. An unknown tier then failed the file upload even with drop_params: true on the row. The row's drop_params now rides along to the mapper, and the passthrough branch keeps it out of the modelInput sent to Bedrock. --- litellm/llms/bedrock/files/transformation.py | 8 +++-- .../test_bedrock_files_transformation.py | 32 ++++++++++++------- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 7d3ac50faa0..f0640410b4b 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -942,7 +942,11 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): model), so `input`, `instructions`, `max_output_tokens` and the tool params translate identically in batch and real time. """ - return responses_batch_body_to_chat_body(openai_request_body) + chat_body: Final = responses_batch_body_to_chat_body(openai_request_body) + drop_params: Final = openai_request_body.get("drop_params") + if drop_params is None: + return chat_body + return _frozen_mapping(chain(chat_body.items(), (("drop_params", drop_params),))) @staticmethod def _transform_batch_body_to_chat_body( @@ -1038,7 +1042,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): # like openai.gpt-oss-*, qwen, deepseek, etc.) --- return { "messages": messages, - **optional_params, + **{k: v for k, v in optional_params.items() if k != "drop_params"}, } def _resolve_batch_record_model_and_provider( diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index 5dc242386d3..d82d01fb6b2 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -102,16 +102,15 @@ class TestBedrockFilesTransformation: assert result[0]["modelInput"]["max_tokens"] == 10 @staticmethod - def _nova_row(**body_params: object) -> dict[str, object]: + def _nova_row( + url: str = "/v1/chat/completions", model: str = "us.amazon.nova-pro-v1:0", **body_params: object + ) -> dict[str, object]: + prompt: Final = {"input": "hi"} if url == "/v1/responses" else {"messages": [{"role": "user", "content": "hi"}]} return { "custom_id": "nova-tier-1", "method": "POST", - "url": "/v1/chat/completions", - "body": { - "model": "us.amazon.nova-pro-v1:0", - "messages": [{"role": "user", "content": "hi"}], - **body_params, - }, + "url": url, + "body": {"model": model, **prompt, **body_params}, } def test_nova_row_with_null_service_tier_sends_no_tier(self): @@ -121,18 +120,29 @@ class TestBedrockFilesTransformation: assert "serviceTier" not in result[0]["modelInput"] - def test_nova_row_with_unmapped_service_tier_fails_the_row_like_real_time(self): + @pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"]) + def test_nova_row_with_unmapped_service_tier_fails_the_row_like_real_time(self, url: str): with pytest.raises(litellm.UnsupportedParamsError, match="ultrafast"): BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( - [self._nova_row(service_tier="ultrafast")] + [self._nova_row(url, service_tier="ultrafast")] ) - def test_nova_row_level_drop_params_drops_the_unmapped_service_tier(self): + @pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"]) + def test_nova_row_level_drop_params_drops_the_unmapped_service_tier(self, url: str): result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( - [self._nova_row(service_tier="ultrafast", drop_params=True)] + [self._nova_row(url, service_tier="ultrafast", drop_params=True)] ) assert "serviceTier" not in result[0]["modelInput"] + assert "drop_params" not in json.dumps(result[0]["modelInput"]) + + @pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"]) + def test_passthrough_row_keeps_row_level_drop_params_out_of_the_model_input(self, url: str): + result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( + [self._nova_row(url, "openai.gpt-oss-120b-1:0", drop_params=True)] + ) + + assert "drop_params" not in result[0]["modelInput"] def test_nova_text_only_uses_converse_format(self): """