From f5ef14308bf7582af365aabae1ea0ff1b0c626df Mon Sep 17 00:00:00 2001 From: quietjules <349048734+quietjules@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:33:57 -0800 Subject: [PATCH] fix(bedrock): answer 400 for service tiers Bedrock cannot serve, forward reserved (#50516) * fix(bedrock): answer 400 for service tiers Bedrock cannot serve, forward reserved Converse used to drop any service_tier outside default, flex, and priority with no error, rewrite auto to default, and drop reserved. Now reserved and standard_only map to Bedrock tiers, auto sends no tier, and any other value answers 400 unless drop_params is on. Nova batch rows honor the row's drop_params. * fix(bedrock): keep a Responses batch row's drop_params and keep it out of passthrough rows * test(bedrock): annotate service tier test locals as Final and drop the row helper's kwargs * fix(bedrock): refuse a service tier on Nova batch rows unless drop_params, and honor the deployment's drop_params * fix(bedrock): map standard and any-case service tiers, and drop tiers from Claude batch rows under the deployment's drop_params * fix(bedrock): forward the tiers OpenAI-format batch rows take, accept auto on Claude rows, and label tiers in lowercase * fix(bedrock): mark the service tier block field read-only * fix(bedrock): read drop_params given as a string on batch rows, and cover the Nova row mapper's drop_params * test(prometheus): sort the service tier label test imports * fix(bedrock): send the Invoke route's service tier as a header and name the accepted tiers in the 400 * fix(bedrock): report and bill the tier Bedrock served on Invoke calls and streams, and send the tier header only to Nova * fix(bedrock): return copies when stamping the served tier, read its header without re-encoding the others, and pin it per stream chunk The Nova Invoke mapper and the event-stream decoder now return model_copy results instead of writing service_tier in place. One shared reader finds the X-Amzn-Bedrock-Service-Tier response header case-insensitively, so a non-ASCII request id no longer breaks the stream decoder, and the passthrough uses the same reader. Tests now vary the served tier in both directions, check every chunk, and cover a stream with no tier header or an empty one * test(bedrock): check the served tier on a stream whose headers come as a plain dict with an upper-case name The non-ASCII request id stream test now also carries X-AMZN-BEDROCK-SERVICE-TIER in a plain dict and asserts every chunk reports that tier, so a case-sensitive header reader or a lookup by the canonical name fails it * refactor(bedrock): drop the passthrough's own service tier header reader that main re-added The passthrough already reads the served tier through the shared case-insensitive reader in common_utils, so the httpx.Headers copy main brought back with the embeddings change has no callers and references a constant this branch removed --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../litellm_core_utils/service_tier_utils.py | 21 +- .../bedrock/chat/converse_transformation.py | 46 +++- litellm/llms/bedrock/chat/invoke_handler.py | 18 +- .../amazon_nova_transformation.py | 7 +- .../base_invoke_transformation.py | 24 +- litellm/llms/bedrock/common_utils.py | 8 + litellm/llms/bedrock/files/transformation.py | 90 ++++++- .../bedrock/passthrough/transformation.py | 13 +- litellm/types/llms/bedrock.py | 2 +- .../test_prometheus_service_tier_label.py | 18 +- .../test_amazon_nova_transformation.py | 65 +++++ .../test_base_invoke_transformation.py | 27 +++ .../llms/bedrock/chat/test_invoke_handler.py | 122 +++++++++- .../llms/bedrock/chat/test_service_tier.py | 162 ++++++++++++- .../test_bedrock_files_transformation.py | 229 ++++++++++++++++++ 15 files changed, 786 insertions(+), 66 deletions(-) diff --git a/litellm/litellm_core_utils/service_tier_utils.py b/litellm/litellm_core_utils/service_tier_utils.py index 04a25982886..a06c9629981 100644 --- a/litellm/litellm_core_utils/service_tier_utils.py +++ b/litellm/litellm_core_utils/service_tier_utils.py @@ -5,13 +5,14 @@ from typing import Final from litellm.types.utils import ServiceTier, StandardLoggingPayload # Tiers a caller may name in a request, across the providers that accept the -# parameter: OpenAI ("auto", "default", "flex", "priority", "scale"), Bedrock and -# Groq (subsets of those), Anthropic ("auto", "standard_only") and Vertex, which -# maps "default" to "standard". Bounds the caller-controlled requested tier +# parameter: OpenAI ("auto", "default", "flex", "priority", "scale"), Bedrock +# ("auto", "default", "flex", "priority", "reserved", "standard", "standard_only"), +# Groq (a subset of OpenAI's), Anthropic ("auto", "standard_only") and Vertex, +# which maps "default" to "standard". Bounds the caller-controlled requested tier # wherever it is recorded. Derived from ``ServiceTier`` so a tier added there for # cost calculation cannot go missing here. KNOWN_REQUEST_SERVICE_TIERS: Final = frozenset( - tuple(tier.value for tier in ServiceTier) + ("batch", "default", "scale", "standard", "standard_only") + tuple(tier.value for tier in ServiceTier) + ("batch", "default", "reserved", "scale", "standard", "standard_only") ) @@ -20,8 +21,7 @@ def get_served_service_tier(standard_logging_payload: StandardLoggingPayload) -> The tier the provider reports it actually served the request on. Providers report it either at the top level of the response (OpenAI, Bedrock, - Groq) or on the usage object (Anthropic). Streaming responses carry no served - tier. + Groq) or on the usage object (Anthropic). """ response: Final = standard_logging_payload.get("response") usage_object: Final = standard_logging_payload.get("metadata", {}).get("usage_object") @@ -38,13 +38,14 @@ def get_requested_service_tier(standard_logging_payload: StandardLoggingPayload) The tier the caller asked for, as sent to the provider. The value is caller-controlled and survives param mapping even where the - provider then ignores it (Bedrock and Groq accept the request and drop an - unrecognized tier), so it is only reported when it names a known tier. + provider then drops it (Groq for an unrecognized tier, Bedrock for one it + cannot serve under ``drop_params``), so it is only reported when it names a + known tier, in lowercase since Bedrock takes any case. """ model_parameters: Final = standard_logging_payload.get("model_parameters") requested_tier: Final = model_parameters.get("service_tier") if isinstance(model_parameters, dict) else None - if isinstance(requested_tier, str) and requested_tier in KNOWN_REQUEST_SERVICE_TIERS: - return requested_tier + if isinstance(requested_tier, str) and requested_tier.lower() in KNOWN_REQUEST_SERVICE_TIERS: + return requested_tier.lower() return None diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 5c82a8d6bc4..12915728b19 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -202,9 +202,16 @@ def _tools_the_model_accepts( _OUTPUT_CONFIG_BLOCK: Final = TypeAdapter(OutputConfigBlock) _DEEPSEEK_THINKING_REASON: Final = "DeepSeek on Bedrock reasons natively and takes no thinking field." +BEDROCK_SERVICE_TIERS: Final = ("default", "flex", "priority", "reserved") +_BEDROCK_SERVICE_TIER_ALIASES: Final = types.MappingProxyType({"standard": "default", "standard_only": "default"}) _DEEPSEEK_R1_EFFORT_REASON: Final = "DeepSeek R1 on Bedrock always reasons and rejects reasoning_effort." +def bedrock_service_tier_name(requested_tier: str) -> str: + tier: Final = requested_tier.lower() + return _BEDROCK_SERVICE_TIER_ALIASES.get(tier, tier) + + def _has_native_text_format(output_config: object) -> bool: if not isinstance(output_config, Mapping): return False @@ -1253,8 +1260,13 @@ class AmazonConverseConfig(BaseConfig): self._map_context_management_param(value, optional_params) if param == "requestMetadata": self._map_request_metadata_param(value, optional_params, drop_params) - if param == "service_tier" and isinstance(value, str): - self._map_service_tier_param(value, optional_params) + if param == "service_tier": + self._map_service_tier_param( + REQUEST_PARAMS.validate_python(request_params)["service_tier"], + optional_params, + model, + drop_params, + ) if param == "web_search_options" and isinstance(value, dict): # Note: we use `isinstance(value, dict)` instead of `value and isinstance(value, dict)` @@ -1308,17 +1320,25 @@ class AmazonConverseConfig(BaseConfig): if mapped is not None: optional_params["context_management"] = mapped - def _map_service_tier_param(self, value: str, optional_params: dict) -> None: - """Map OpenAI service_tier (string) to Bedrock serviceTier (object). - - OpenAI values: "auto", "default", "flex", "priority" - Bedrock values: "default", "flex", "priority" (no "auto") - """ - bedrock_tier = value - if value == "auto": - bedrock_tier = "default" # Bedrock doesn't support "auto" - if bedrock_tier in ("default", "flex", "priority"): - optional_params["serviceTier"] = {"type": bedrock_tier} + def _map_service_tier_param(self, value: object, optional_params: dict, model: str, drop_params: bool) -> None: + tier: Final = bedrock_service_tier_name(value) if isinstance(value, str) else None + if value is None or tier == "auto": + return + if tier in BEDROCK_SERVICE_TIERS: + optional_params["serviceTier"] = {"type": tier} + return + if not (litellm.drop_params or drop_params): + raise litellm.utils.UnsupportedParamsError( + message=( + f"{model} does not support service_tier={value!r} on this Bedrock route, which takes " + f"auto, standard, or one of {', '.join(BEDROCK_SERVICE_TIERS)}. " + "To drop unsupported params, set `litellm.drop_params = True`." + ), + status_code=400, + ) + verbose_logger.warning( + "Dropping service_tier=%r for %s, since this Bedrock route does not serve it.", value, model + ) def _translate_response_format_param( self, diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 57cc39a03f4..785e1742029 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -1,7 +1,7 @@ import json import types from collections.abc import AsyncIterator, Iterator, Mapping -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, TypeVar import httpx from pydantic import TypeAdapter, ValidationError @@ -61,6 +61,7 @@ from ..common_utils import ( get_bedrock_stream_event_statuses, get_bedrock_tool_name, parse_bedrock_event_stream, + served_service_tier, ) bedrock_tool_name_mappings: Final[InMemoryCache] = InMemoryCache(max_size_in_memory=50, default_ttl=600) @@ -218,6 +219,15 @@ def _stream_decoder( return AWSEventStreamDecoder(model=model, json_mode=json_mode) +_StreamChunk: Final = TypeVar("_StreamChunk") + + +def _with_served_service_tier(chunk: _StreamChunk, served_tier: str | None) -> _StreamChunk: + if served_tier is None or not isinstance(chunk, ModelResponseStream): + return chunk + return chunk.model_copy(update={"service_tier": served_tier}) + + async def make_call( client: AsyncHTTPHandler | None, api_base: str, @@ -851,6 +861,7 @@ class AWSEventStreamDecoder: event_stream_buffer: Final = EventStreamBuffer() tally: Final = _EventStreamTally(get_bedrock_stream_event_statuses()) request_id_headers: Final = provider_request_id_headers(response_headers) + served_tier: Final = served_service_tier(response_headers) for chunk in iterator: event_stream_buffer.add_data(chunk) tally.add_chunk(chunk) @@ -859,7 +870,7 @@ class AWSEventStreamDecoder: if message: # sse_event = ServerSentEvent(data=message, event="completion") _data = json.loads(message) - yield self.chunk_parser(chunk_data=_data) + yield _with_served_service_tier(self.chunk_parser(chunk_data=_data), served_tier) undecoded_stream_error: Final = tally.undecoded_stream_error(response_headers) if undecoded_stream_error is not None: raise undecoded_stream_error @@ -876,6 +887,7 @@ class AWSEventStreamDecoder: event_stream_buffer: Final = EventStreamBuffer() tally: Final = _EventStreamTally(get_bedrock_stream_event_statuses()) request_id_headers: Final = provider_request_id_headers(response_headers) + served_tier: Final = served_service_tier(response_headers) async for chunk in iterator: event_stream_buffer.add_data(chunk) tally.add_chunk(chunk) @@ -883,7 +895,7 @@ class AWSEventStreamDecoder: message = self._decode_event(event, tally, request_id_headers) if message: _data = json.loads(message) - yield self.chunk_parser(chunk_data=_data) + yield _with_served_service_tier(self.chunk_parser(chunk_data=_data), served_tier) undecoded_stream_error: Final = tally.undecoded_stream_error(response_headers) if undecoded_stream_error is not None: raise undecoded_stream_error diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py index ce28e579dcf..23f8de49781 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py @@ -24,6 +24,7 @@ from litellm.types.llms.bedrock import ( from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse +from ...common_utils import served_service_tier from ..converse_transformation import AmazonConverseConfig from .base_invoke_transformation import AmazonInvokeConfig @@ -142,7 +143,7 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig): api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: - return AmazonConverseConfig.transform_response( + response: Final = AmazonConverseConfig.transform_response( self, model, raw_response, @@ -156,6 +157,10 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig): api_key, json_mode, ) + served_tier: Final = served_service_tier(raw_response.headers) + if served_tier is None: + return response + return response.model_copy(update={"service_tier": served_tier}) @staticmethod def _inline_cache_points(request: BedrockInvokeNovaRequest) -> BedrockInvokeNovaRequest: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 877fb4ca692..f5e55eea141 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -22,7 +22,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.bedrock.chat.invoke_handler import make_call, make_sync_call -from litellm.llms.bedrock.common_utils import BedrockError +from litellm.llms.bedrock.common_utils import BEDROCK_SERVICE_TIER_HEADER, BedrockError from litellm.llms.bedrock.request_metadata import ( bedrock_request_metadata_headers, merge_bedrock_invoke_headers, @@ -67,8 +67,13 @@ class _LlamaInvokeResponse(TypedDict): generation: ReadOnly[str | None] +class _InvokeServiceTierParams(TypedDict): + serviceTier: NotRequired[ReadOnly[object]] + + _COHERE_INVOKE_RESPONSE: Final = TypeAdapter(_CohereInvokeResponse) _LLAMA_INVOKE_RESPONSE: Final = TypeAdapter(_LlamaInvokeResponse) +_INVOKE_SERVICE_TIER_PARAMS: Final = TypeAdapter(_InvokeServiceTierParams) _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = ( "{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}" @@ -79,6 +84,14 @@ def _response_header(headers: Mapping[str, str], name: str) -> str | None: return headers.get(name, None) +def _bedrock_invoke_service_tier_headers(optional_params: _InvokeServiceTierParams) -> tuple[tuple[str, str], ...]: + match optional_params.get("serviceTier"): + case {"type": str(tier)}: + return ((BEDROCK_SERVICE_TIER_HEADER, tier),) + case _: + return () + + def bedrock_invoke_guardrail_headers(raw_guardrail_config: object) -> "dict[str, str]": try: guardrail_config: Final = _GUARDRAIL_CONFIG_VALIDATOR.validate_python(raw_guardrail_config) @@ -455,8 +468,15 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): if raw_guardrail_config is None else tuple(bedrock_invoke_guardrail_headers(raw_guardrail_config).items()) ) + service_tier_headers: Final = ( + _bedrock_invoke_service_tier_headers(_INVOKE_SERVICE_TIER_PARAMS.validate_python(optional_params)) + if self.get_bedrock_invoke_provider(model) == "nova" + else () + ) owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params) - return merge_bedrock_invoke_headers(headers, guardrail_headers, metadata_headers, owned_names) + return merge_bedrock_invoke_headers( + headers, (*guardrail_headers, *service_tier_headers), metadata_headers, owned_names + ) def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: return BedrockError(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 6a05cb1d834..c25ef76c925 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -58,6 +58,7 @@ if TYPE_CHECKING: from litellm.types.llms.openai import AllMessageValues BEDROCK_INVOKE_TRACE_FIELD: Final = "amazon-bedrock-trace" +BEDROCK_SERVICE_TIER_HEADER: Final = "X-Amzn-Bedrock-Service-Tier" BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS: Final[Mapping[str, str]] = MappingProxyType( { "compact_20260112": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, @@ -86,6 +87,13 @@ BedrockRoute = Literal[ ] +def served_service_tier(response_headers: Mapping[str, str] | None) -> str | None: + tier_header: Final = BEDROCK_SERVICE_TIER_HEADER.lower() + return next( + (value for name, value in (response_headers or {}).items() if name.lower() == tier_header and value), None + ) + + def error_response_text(response: httpx.Response) -> str: try: return response.text diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index b19ad776051..5da1e582df8 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -24,7 +24,7 @@ from typing_extensions import TypedDict as ExtraItemsTypedDict from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.constants import BEDROCK_INVOKE_PROVIDERS_LITERAL -from litellm.exceptions import BadRequestError +from litellm.exceptions import BadRequestError, UnsupportedParamsError from litellm.files.utils import FilesAPIUtils from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.cloud_storage_security import ( @@ -40,7 +40,7 @@ from litellm.litellm_core_utils.cloud_storage_security import ( split_configured_cloud_bucket_name, validate_managed_cloud_file_id, ) -from litellm.litellm_core_utils.core_helpers import require_attributes +from litellm.litellm_core_utils.core_helpers import normalize_drop_params, require_attributes from litellm.litellm_core_utils.prompt_templates.common_utils import ( text_completion_prompt_to_messages, ) @@ -49,7 +49,7 @@ from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( ) from litellm.llms.anthropic.common_utils import AnthropicError from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role -from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS, BaseLLMException from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body from litellm.llms.base_llm.files.transformation import ( BaseFilesConfig, @@ -391,6 +391,42 @@ def _batch_line_error(line_number: int, problem: str) -> BedrockError: return _client_error(f"Batch input file line {line_number} {problem}") +_OPENAI_FORMAT_INVOKE_SERVICE_TIERS: Final = frozenset({"default", "flex", "priority"}) + + +def _check_batch_record_service_tier( + value: object, model: str, drop_params: bool, accepted_tiers: frozenset[str] = frozenset() +) -> None: + import litellm + + if value is None or (isinstance(value, str) and value.lower() == "auto"): + return + if not (litellm.drop_params or drop_params): + raise UnsupportedParamsError( + message=( + f"{model} batch records do not support service_tier={value!r}. " + + ( + f"Their InvokeModel body takes only {', '.join(sorted(accepted_tiers))}. " + if accepted_tiers + else "A Bedrock batch record for this model is an InvokeModel body, which takes no service tier. " + ) + + "To drop unsupported params, set `litellm.drop_params = True`." + ), + status_code=400, + ) + verbose_logger.warning("Dropping service_tier=%r from a %s batch record, since Bedrock rejects it.", value, model) + + +def _openai_format_batch_record_service_tier(value: object, model: str, drop_params: bool) -> Mapping[str, str]: + from litellm.llms.bedrock.chat.converse_transformation import bedrock_service_tier_name + + tier: Final = bedrock_service_tier_name(value) if isinstance(value, str) else None + if tier in _OPENAI_FORMAT_INVOKE_SERVICE_TIERS: + return MappingProxyType({"service_tier": tier}) + _check_batch_record_service_tier(value, model, drop_params, _OPENAI_FORMAT_INVOKE_SERVICE_TIERS) + return MappingProxyType({}) + + def _decode_batch_input(content: str | bytes) -> str: if isinstance(content, str): return content.removeprefix("\ufeff") @@ -1047,7 +1083,11 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): model), so `input`, `instructions`, `max_output_tokens` and the tool params translate identically in batch and real time. """ - return responses_batch_body_to_chat_body(openai_request_body) + chat_body: Final = responses_batch_body_to_chat_body(openai_request_body) + drop_params: Final = openai_request_body.get("drop_params") + if drop_params is None: + return chat_body + return _frozen_mapping(chain(chat_body.items(), (("drop_params", drop_params),))) @staticmethod def _transform_batch_body_to_chat_body( @@ -1073,6 +1113,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): openai_request_body: Mapping[str, Any], model: str, provider: str | None = None, + drop_params: bool = False, model_info: Mapping[str, object] | None = None, ) -> dict[str, object]: """ @@ -1088,6 +1129,10 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): messages: Final = map_developer_role_to_system_role(openai_request_body.get("messages", [])) request_params: Final[Mapping[str, object]] = openai_request_body optional_params: Final = {k: v for k, v in request_params.items() if k not in ["model", "messages"]} + drop_unsupported_params: Final = ( + drop_params or normalize_drop_params(optional_params.get("drop_params")) is True + ) + requested_service_tier: Final = REQUEST_PARAMS.validate_python(optional_params).get("service_tier") # --- Anthropic: use existing AmazonAnthropicClaudeConfig --- if provider == LlmProviders.ANTHROPIC: @@ -1095,6 +1140,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): AmazonAnthropicClaudeConfig, ) + _check_batch_record_service_tier(requested_service_tier, model, drop_unsupported_params) config: Final = AmazonAnthropicClaudeConfig() litellm_params: Final = without_bedrock_native_output_opt_in( { @@ -1114,8 +1160,10 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): **{ k: v for k, v in optional_params.items() - if not is_litellm_owned_kwarg(k) or k in _LITELLM_PARAMS_THE_MAPPER_TAKES + if k != "service_tier" + and (not is_litellm_owned_kwarg(k) or k in _LITELLM_PARAMS_THE_MAPPER_TAKES) } + | ({"drop_params": True} if drop_params else {}) ), ) return config.transform_request( @@ -1136,12 +1184,15 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): AmazonConverseConfig, ) + _check_batch_record_service_tier(requested_service_tier, model, drop_unsupported_params) converse_config: Final = AmazonConverseConfig() mapped_params = converse_config.map_openai_params( - non_default_params={k: v for k, v in optional_params.items() if k != "requestMetadata"}, + non_default_params={ + k: v for k, v in optional_params.items() if k not in ("requestMetadata", "service_tier") + }, optional_params={}, model=model, - drop_params=False, + drop_params=drop_unsupported_params, ) return converse_config.transform_request( model=model, @@ -1155,7 +1206,8 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): # like openai.gpt-oss-*, qwen, deepseek, etc.) --- return { "messages": messages, - **optional_params, + **{k: v for k, v in optional_params.items() if k not in ("drop_params", "service_tier")}, + **_openai_format_batch_record_service_tier(requested_service_tier, model, drop_unsupported_params), } def _resolve_batch_record_model_and_provider( @@ -1203,17 +1255,25 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): batch_input_lines: Sequence[_BatchInputLine], target_model: str = "", model_info: Mapping[str, object] | None = None, + drop_params: bool = False, ) -> tuple[_BedrockBatchRecord, ...]: return tuple( - self._transform_batch_input_line(idx, batch_input_line, target_model, model_info) + self._transform_batch_input_line(idx, batch_input_line, target_model, model_info, drop_params) for idx, batch_input_line in enumerate(batch_input_lines) ) def _transform_batch_input_line( - self, idx: int, batch_input_line: _BatchInputLine, target_model: str, model_info: Mapping[str, object] | None + self, + idx: int, + batch_input_line: _BatchInputLine, + target_model: str, + model_info: Mapping[str, object] | None, + drop_params: bool, ) -> _BedrockBatchRecord: try: - return self._transform_openai_batch_record(idx, batch_input_line.record, target_model, model_info) + return self._transform_openai_batch_record( + idx, batch_input_line.record, target_model, model_info, drop_params + ) except _BATCH_RECORD_CONVERSION_ERRORS as e: raise _batch_line_error( batch_input_line.line_number, f"could not be converted to a Bedrock batch record: {e}" @@ -1225,6 +1285,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): openai_record: _OpenAIBatchRecord, target_model: str, model_info: Mapping[str, object] | None = None, + drop_params: bool = False, ) -> _BedrockBatchRecord: import litellm @@ -1249,6 +1310,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): model=model_for_transform, provider=provider, model_info=model_info, + drop_params=drop_params, ) ) return _BedrockBatchRecord( @@ -1293,8 +1355,12 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): batch_input_lines: Final = _parse_batch_jsonl(original_file_content) litellm_params_model: Final = litellm_params.get("model") target_model: Final = model or (litellm_params_model if isinstance(litellm_params_model, str) else "") + deployment_drop_params: Final = REQUEST_PARAMS.validate_python(litellm_params).get("drop_params") bedrock_jsonl_content: Final = self.transform_batch_input_lines( - batch_input_lines, target_model=target_model, model_info=_upload_deployment_model_info(litellm_params) + batch_input_lines, + target_model=target_model, + model_info=_upload_deployment_model_info(litellm_params), + drop_params=normalize_drop_params(deployment_drop_params) is True, ) file_content = "\n".join(json.dumps(item) for item in bedrock_jsonl_content) elif isinstance(extracted_file_data_content, bytes): diff --git a/litellm/llms/bedrock/passthrough/transformation.py b/litellm/llms/bedrock/passthrough/transformation.py index ad6586f6043..6e4868ab271 100644 --- a/litellm/llms/bedrock/passthrough/transformation.py +++ b/litellm/llms/bedrock/passthrough/transformation.py @@ -31,6 +31,7 @@ from ..common_utils import ( BedrockError, BedrockEventStreamDecoderBase, BedrockModelInfo, + served_service_tier, ) from ..request_metadata import ( BEDROCK_REQUEST_METADATA_HEADER, @@ -59,7 +60,6 @@ _BEDROCK_INVOKE_ACTIONS: Final = frozenset({"invoke", "invoke-with-response-stre _BEDROCK_INVOKE_GUARDRAIL_HEADERS: Final = frozenset( {"x-amzn-bedrock-guardrailidentifier", "x-amzn-bedrock-guardrailversion", "x-amzn-bedrock-trace"} ) -_BEDROCK_SERVICE_TIER_HEADER: Final = "x-amzn-bedrock-service-tier" _NO_RESPONSE_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) _TieredResponse: Final = TypeVar("_TieredResponse", bound=BaseModel) @@ -423,13 +423,6 @@ def _invoke_embedding_response(model: str, httpx_response: Response, has_input_i ) -def _served_service_tier(response_headers: Mapping[str, str]) -> str | None: - headers: Final = httpx.Headers(response_headers) - if _BEDROCK_SERVICE_TIER_HEADER not in headers: - return None - return headers[_BEDROCK_SERVICE_TIER_HEADER] - - def _with_served_service_tier(response: _TieredResponse, service_tier: str | None) -> _TieredResponse: if service_tier is None: return response @@ -659,7 +652,7 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD if "invoke" in endpoint: logging_obj.record_native_bedrock_guardrail(_invoke_guardrail_trace(httpx_response)) - return _with_served_service_tier(litellm_model_response, _served_service_tier(httpx_response.headers)) + return _with_served_service_tier(litellm_model_response, served_service_tier(httpx_response.headers)) def create_stream_collector( self, @@ -672,7 +665,7 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD return BedrockEventStreamCollector( parse_event=self._parse_message_from_event, decoder=self._get_event_stream_decoder(model=model, endpoint=endpoint), - service_tier=_served_service_tier(response_headers), + service_tier=served_service_tier(response_headers), ) def _get_event_stream_decoder(self, model: str, endpoint: str) -> Optional["AWSEventStreamDecoder"]: diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index 170f90c1adb..c721208808d 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -244,7 +244,7 @@ class ConverseTokenUsageBlock(TypedDict, total=False): class ServiceTierBlock(TypedDict): - type: Literal["priority", "default", "flex"] + type: ReadOnly[Literal["priority", "default", "flex", "reserved"]] class ConverseResponseBlock(TypedDict, total=False): diff --git a/tests/unit/integrations/test_prometheus_service_tier_label.py b/tests/unit/integrations/test_prometheus_service_tier_label.py index 8b8131b5af2..89ed73b25c1 100644 --- a/tests/unit/integrations/test_prometheus_service_tier_label.py +++ b/tests/unit/integrations/test_prometheus_service_tier_label.py @@ -10,6 +10,7 @@ Run with: """ import datetime +from typing import Final import pytest @@ -18,6 +19,7 @@ from litellm.litellm_core_utils.service_tier_utils import ( KNOWN_REQUEST_SERVICE_TIERS, get_service_tier_from_standard_logging_payload, ) +from litellm.llms.bedrock.chat.converse_transformation import BEDROCK_SERVICE_TIERS from litellm.types.integrations.prometheus import ( PrometheusMetricLabels, UserAPIKeyLabelNames, @@ -138,7 +140,7 @@ def test_requested_tier_used_when_no_served_tier(): def test_unrecognized_requested_tier_is_not_labelled(): """ A caller-supplied tier survives param mapping even where the provider then - ignores it (Bedrock and Groq drop an unrecognized tier and still answer), so + ignores it (Groq drops an unrecognized tier and still answers), so labelling it verbatim would let one caller mint a series per string. """ payload = _standard_logging_payload(model_parameters={"service_tier": "attacker-controlled-a1b2c3"}) @@ -166,6 +168,20 @@ def test_every_known_requested_tier_is_labelled(tier): assert get_service_tier_from_standard_logging_payload(payload) == tier +@pytest.mark.parametrize("tier", BEDROCK_SERVICE_TIERS) +def test_every_tier_bedrock_forwards_is_labelled(tier: str): + payload: Final = _standard_logging_payload(model_parameters={"service_tier": tier}) + + assert get_service_tier_from_standard_logging_payload(payload) == tier + + +@pytest.mark.parametrize("tier", ["Priority", "FLEX", "Reserved"]) +def test_a_requested_tier_in_any_case_is_labelled_in_lowercase(tier: str): + payload: Final = _standard_logging_payload(model_parameters={"service_tier": tier}) + + assert get_service_tier_from_standard_logging_payload(payload) == tier.lower() + + @pytest.mark.parametrize( "response, usage_object, model_parameters", [ diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_amazon_nova_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_amazon_nova_transformation.py index de4b135e6d0..a4109b6b325 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_amazon_nova_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_amazon_nova_transformation.py @@ -1,11 +1,16 @@ import json +from collections.abc import Mapping +from typing import Final +from unittest.mock import MagicMock +import httpx import pytest import litellm from litellm.llms.bedrock.chat.invoke_transformations.amazon_nova_transformation import ( AmazonInvokeNovaConfig, ) +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.types.integrations.anthropic_cache_control_hook import GATEWAY_INJECTED_CACHE_METADATA_KEY MODEL = "us.amazon.nova-pro-v1:0" @@ -134,3 +139,63 @@ def test_caller_request_metadata_is_ignored_because_invoke_model_has_no_field_fo assert "requestMetadata" not in request assert request["inferenceConfig"] == {"maxTokens": 8} + + +@pytest.mark.parametrize( + ("service_tier", "sent_tier"), + [("priority", "priority"), ("flex", "flex"), ("reserved", "reserved"), ("Standard", "default"), ("auto", None)], +) +def test_invoke_sends_the_service_tier_in_the_header_invoke_model_takes(service_tier: str, sent_tier: str | None): + send: Final = MagicMock(return_value=httpx.Response(200)) + client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(send))) + + litellm.completion( + model=f"bedrock/invoke/{MODEL}", + messages=[{"role": "user", "content": "hi"}], + stream=True, + service_tier=service_tier, + client=client, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + ) + + request: Final = send.call_args.args[0] + assert request.headers.get("X-Amzn-Bedrock-Service-Tier") == sent_tier + assert "serviceTier" not in json.loads(request.content) + + +@pytest.mark.parametrize(("requested_tier", "served_tier"), [("priority", "flex"), ("flex", "priority")]) +def test_invoke_reports_and_bills_the_tier_bedrock_served_over_the_tier_requested( + requested_tier: str, served_tier: str +) -> None: + reply: Final = { + "output": {"message": {"role": "assistant", "content": [{"text": "hi"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1000, "outputTokens": 1000, "totalTokens": 2000}, + } + + def invoke(tier: Mapping[str, object]) -> litellm.ModelResponse: + send: Final = MagicMock( + return_value=httpx.Response(200, json=reply, headers={"X-Amzn-Bedrock-Service-Tier": served_tier}) + ) + return litellm.completion( + model=f"bedrock/invoke/{MODEL}", + messages=[{"role": "user", "content": "hi"}], + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(send))), + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + **tier, + ) + + native_requested: Final = invoke({"serviceTier": {"type": requested_tier}}) + openai_requested: Final = invoke({"service_tier": requested_tier}) + openai_served: Final = invoke({"service_tier": served_tier}) + + assert native_requested.service_tier == openai_requested.service_tier == served_tier + assert ( + native_requested._hidden_params["response_cost"] + == openai_requested._hidden_params["response_cost"] + == openai_served._hidden_params["response_cost"] + ) diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py index f6728e2ed10..dc84052b812 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py @@ -654,3 +654,30 @@ def test_guarded_non_claude_invoke_stream_yields_only_the_text_and_stop_chunks( ([(choice.delta.content, choice.finish_reason) for choice in chunk.choices], chunk.provider_specific_fields or {}) for chunk in stream ] == [([("hi", None)], {}), ([(None, "stop")], {})] + + +@pytest.mark.parametrize( + "model", + [ + "anthropic.claude-sonnet-4-6", + "amazon.titan-text-express-v1", + "mistral.mistral-7b-instruct-v0:2", + ], +) +def test_a_native_service_tier_sends_no_tier_header_to_invoke_models_whose_replies_do_not_report_it(model: str) -> None: + send: Final = MagicMock(return_value=httpx.Response(200)) + client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(send))) + + litellm.completion( + model=f"bedrock/invoke/{model}", + messages=[{"role": "user", "content": "hi"}], + stream=True, + max_tokens=10, + client=client, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + serviceTier={"type": "priority"}, + ) + + assert "X-Amzn-Bedrock-Service-Tier" not in send.call_args.args[0].headers diff --git a/tests/unit/llms/bedrock/chat/test_invoke_handler.py b/tests/unit/llms/bedrock/chat/test_invoke_handler.py index 2faa089e1c3..3a7a4273d46 100644 --- a/tests/unit/llms/bedrock/chat/test_invoke_handler.py +++ b/tests/unit/llms/bedrock/chat/test_invoke_handler.py @@ -1539,16 +1539,23 @@ async def test_aiter_bytes_stream_error_carries_only_the_provider_request_id_hea @pytest.mark.parametrize("request_id", ("req-\u00fc-1", "req-\udcfc-1"), ids=("latin-1", "surrogate-escaped")) -def test_iter_bytes_streams_and_fails_normally_with_a_non_ascii_provider_request_id(request_id: str) -> None: +def test_iter_bytes_streams_the_served_tier_and_fails_normally_with_a_non_ascii_provider_request_id( + request_id: str, +) -> None: decoder: Final = AWSEventStreamDecoder(model="us.moonshotai.kimi-k3") - headers: Final = {"content-type": "application/vnd.amazon.eventstream", "x-amzn-RequestId": request_id} + headers: Final = { + "content-type": "application/vnd.amazon.eventstream", + "x-amzn-RequestId": request_id, + "X-AMZN-BEDROCK-SERVICE-TIER": "flex", + } chunks: Final = list(decoder.iter_bytes(iter(_CONVERSE_CONTENT_FRAMES), response_headers=headers)) with pytest.raises(BedrockError) as exc_info: list(decoder.iter_bytes(iter([_unknown_event_frame()]), response_headers=headers)) - texts: Final = [chunk.choices[0].delta.content for chunk in chunks if isinstance(chunk, ModelResponseStream)] - assert "".join(text or "" for text in texts) == "hi" + stream_chunks: Final = [chunk for chunk in chunks if isinstance(chunk, ModelResponseStream)] + assert "".join(chunk.choices[0].delta.content or "" for chunk in stream_chunks) == "hi" + assert {chunk.service_tier for chunk in stream_chunks} == {"flex"} assert exc_info.value.status_code == 502 assert exc_info.value.headers is None @@ -2084,3 +2091,110 @@ def test_invoke_stream_decoder_carries_the_guardrail_trace_of_the_message_stop_c assert (plain.provider_specific_fields or {}).get("amazon-bedrock-trace") is None assert traced.provider_specific_fields is not None assert traced.provider_specific_fields["amazon-bedrock-trace"] == trace + + +NOVA_INVOKE_STREAM_EVENTS: Final = ( + {"messageStart": {"role": "assistant"}}, + {"contentBlockDelta": {"contentBlockIndex": 0, "delta": {"text": "hi"}}}, + {"contentBlockStop": {"contentBlockIndex": 0}}, + {"messageStop": {"stopReason": "end_turn"}}, + {"metadata": {"usage": {"inputTokens": 1000, "outputTokens": 1000, "totalTokens": 2000}, "metrics": {}}}, +) + + +def _nova_invoke(served_tier: str | None) -> httpx.MockTransport: + tier_header: Final = {} if served_tier is None else {"X-Amzn-Bedrock-Service-Tier": served_tier} + + def reply(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/invoke-with-response-stream"): + return httpx.Response( + 200, + headers={"content-type": "application/vnd.amazon.eventstream", **tier_header}, + content=b"".join(_bedrock_event_stream_frame(event) for event in NOVA_INVOKE_STREAM_EVENTS), + ) + return httpx.Response( + 200, + headers=tier_header, + json={ + "output": {"message": {"role": "assistant", "content": [{"text": "hi"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1000, "outputTokens": 1000, "totalTokens": 2000}, + }, + ) + + return httpx.MockTransport(reply) + + +async def _nova_invoke_calls( + requested_tier: Mapping[str, object], served_tier: str | None +) -> tuple[litellm.ModelResponse, Sequence[ModelResponseStream], Sequence[ModelResponseStream]]: + def completion(stream: bool) -> object: + return litellm.completion( + model="bedrock/invoke/us.amazon.nova-pro-v1:0", + messages=[{"role": "user", "content": "hi"}], + stream=stream, + client=HTTPHandler(client=httpx.Client(transport=_nova_invoke(served_tier))), + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + **requested_tier, + ) + + async_stream: Final = await litellm.acompletion( + model="bedrock/invoke/us.amazon.nova-pro-v1:0", + messages=[{"role": "user", "content": "hi"}], + stream=True, + client=AsyncHTTPHandler(transport=_nova_invoke(served_tier)), + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + **requested_tier, + ) + non_streaming: Final = completion(stream=False) + assert isinstance(non_streaming, litellm.ModelResponse) + return non_streaming, tuple(completion(stream=True)), tuple([chunk async for chunk in async_stream]) + + +@pytest.mark.parametrize( + ("requested_tier", "served_tier"), + [ + pytest.param({"serviceTier": {"type": "priority"}}, "flex", id="native"), + pytest.param({"service_tier": "priority"}, "flex", id="openai"), + pytest.param({"service_tier": "flex"}, "priority", id="openai-served-priority"), + ], +) +@pytest.mark.asyncio +async def test_a_nova_invoke_stream_reports_and_bills_the_tier_bedrock_served_like_a_non_streaming_call( + requested_tier: Mapping[str, object], served_tier: str +) -> None: + non_streaming, sync_chunks, async_chunks = await _nova_invoke_calls(requested_tier, served_tier) + sync_streamed: Final = litellm.stream_chunk_builder(list(sync_chunks)) + async_streamed: Final = litellm.stream_chunk_builder(list(async_chunks)) + + assert {getattr(chunk, "service_tier", None) for chunk in (*sync_chunks, *async_chunks)} == {served_tier} + assert sync_streamed.service_tier == async_streamed.service_tier == non_streaming.service_tier == served_tier + assert ( + litellm.completion_cost(sync_streamed) + == litellm.completion_cost(async_streamed) + == non_streaming._hidden_params["response_cost"] + ) + + +@pytest.mark.parametrize( + "requested_tier", + [ + pytest.param({"serviceTier": {"type": "priority"}}, id="native"), + pytest.param({"service_tier": "priority"}, id="openai"), + ], +) +@pytest.mark.parametrize("served_tier", [pytest.param(None, id="no-header"), pytest.param("", id="empty-header")]) +@pytest.mark.asyncio +async def test_a_nova_invoke_stream_without_a_served_tier_reports_no_tier( + requested_tier: Mapping[str, object], served_tier: str | None +) -> None: + non_streaming, sync_chunks, async_chunks = await _nova_invoke_calls(requested_tier, served_tier) + + assert {getattr(chunk, "service_tier", None) for chunk in (*sync_chunks, *async_chunks)} == {None} + assert getattr(litellm.stream_chunk_builder(list(sync_chunks)), "service_tier", None) is None + assert getattr(litellm.stream_chunk_builder(list(async_chunks)), "service_tier", None) is None + assert getattr(non_streaming, "service_tier", None) is None diff --git a/tests/unit/llms/bedrock/chat/test_service_tier.py b/tests/unit/llms/bedrock/chat/test_service_tier.py index ce9dc4d745e..6957090fcb2 100644 --- a/tests/unit/llms/bedrock/chat/test_service_tier.py +++ b/tests/unit/llms/bedrock/chat/test_service_tier.py @@ -3,11 +3,14 @@ Tests for Bedrock Converse API serviceTier support. """ import json +from typing import Final +from unittest.mock import MagicMock import pytest - +import litellm from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.types.llms.bedrock import ServiceTierBlock @@ -201,24 +204,165 @@ def test_map_openai_service_tier_flex(): assert result["serviceTier"] == {"type": "flex"} -def test_map_openai_service_tier_auto_maps_to_default(): - """Test that OpenAI service_tier='auto' maps to Bedrock serviceTier='default'. +def test_map_openai_service_tier_auto_sends_no_tier(): + """`auto` picks no tier, so Bedrock gets no serviceTier and serves its own default.""" + config: Final = AmazonConverseConfig() - Bedrock doesn't support 'auto', so we map it to 'default'. - """ - config = AmazonConverseConfig() - - result = config.map_openai_params( + result: Final = config.map_openai_params( non_default_params={"service_tier": "auto"}, optional_params={}, model="anthropic.claude-3-sonnet-20240229-v1:0", drop_params=False, ) - assert "serviceTier" in result + assert "serviceTier" not in result + + +def test_map_openai_service_tier_none_sends_no_tier(): + config: Final = AmazonConverseConfig() + + result: Final = config.map_openai_params( + non_default_params={"service_tier": None}, + optional_params={}, + model="anthropic.claude-3-sonnet-20240229-v1:0", + drop_params=False, + ) + + assert "serviceTier" not in result + + +def test_map_openai_service_tier_reserved_reaches_the_request(): + config: Final = AmazonConverseConfig() + model: Final = "anthropic.claude-3-sonnet-20240229-v1:0" + + optional_params: Final = config.map_openai_params( + non_default_params={"service_tier": "reserved"}, + optional_params={}, + model=model, + drop_params=False, + ) + request: Final = config.transform_request( + model=model, + messages=[{"role": "user", "content": "Hello!"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert request["serviceTier"] == {"type": "reserved"} + + +def test_map_openai_service_tier_standard_only_maps_to_default(): + """`standard_only` is the Anthropic Messages value that means Bedrock's Standard tier.""" + config: Final = AmazonConverseConfig() + + result: Final = config.map_openai_params( + non_default_params={"service_tier": "standard_only"}, + optional_params={}, + model="anthropic.claude-3-sonnet-20240229-v1:0", + drop_params=False, + ) + assert result["serviceTier"] == {"type": "default"} +@pytest.mark.parametrize( + ("value", "bedrock_tier"), + [("standard", "default"), ("Standard_Only", "default"), ("Priority", "priority"), ("FLEX", "flex")], +) +def test_map_openai_service_tier_maps_standard_and_any_case_to_the_bedrock_tier(value: str, bedrock_tier: str): + config: Final = AmazonConverseConfig() + + result: Final = config.map_openai_params( + non_default_params={"service_tier": value}, + optional_params={}, + model="anthropic.claude-3-sonnet-20240229-v1:0", + drop_params=False, + ) + + assert result["serviceTier"] == {"type": bedrock_tier} + + +def test_map_openai_service_tier_auto_in_any_case_sends_no_tier(): + config: Final = AmazonConverseConfig() + + result: Final = config.map_openai_params( + non_default_params={"service_tier": "AUTO"}, + optional_params={}, + model="anthropic.claude-3-sonnet-20240229-v1:0", + drop_params=False, + ) + + assert "serviceTier" not in result + + +@pytest.mark.parametrize("value", ["ultrafast", "fast", "scale", "", {"type": "priority"}, ["fast"], 1]) +def test_map_openai_service_tier_unmapped_value_answers_400(value: object): + config: Final = AmazonConverseConfig() + optional_params: Final = dict[str, object]() + + with pytest.raises(litellm.utils.UnsupportedParamsError) as error: + config.map_openai_params( + non_default_params={"service_tier": value}, + optional_params=optional_params, + model="anthropic.claude-3-sonnet-20240229-v1:0", + drop_params=False, + ) + + assert error.value.status_code == 400 + assert f"service_tier={value!r}" in error.value.message + assert "default, flex, priority, reserved" in error.value.message + assert "drop_params" in error.value.message + assert "serviceTier" not in optional_params + + +@pytest.mark.parametrize("value", ["ultrafast", {"type": "priority"}]) +def test_map_openai_service_tier_unmapped_value_drops_under_request_drop_params(value: object): + config: Final = AmazonConverseConfig() + + result: Final = config.map_openai_params( + non_default_params={"service_tier": value}, + optional_params={}, + model="anthropic.claude-3-sonnet-20240229-v1:0", + drop_params=True, + ) + + assert "serviceTier" not in result + + +def test_map_openai_service_tier_unmapped_value_drops_under_global_drop_params(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "drop_params", True) + config: Final = AmazonConverseConfig() + + result: Final = config.map_openai_params( + non_default_params={"service_tier": "ultrafast"}, + optional_params={}, + model="anthropic.claude-3-sonnet-20240229-v1:0", + drop_params=False, + ) + + assert "serviceTier" not in result + + +@pytest.mark.parametrize("model", ["bedrock/converse/us.openai.gpt-6-luna", "bedrock/converse/us.amazon.nova-2-lite-v1:0"]) +def test_completion_refuses_unmapped_service_tier_before_calling_bedrock(model: str, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "drop_params", False) + client: Final = MagicMock(spec=HTTPHandler) + + with pytest.raises(litellm.BadRequestError) as error: + litellm.completion( + model=model, + messages=[{"role": "user", "content": "Hello!"}], + service_tier="ultrafast", + aws_region_name="us-east-1", + client=client, + ) + + assert error.value.status_code == 400 + assert "service_tier='ultrafast'" in error.value.message + client.post.assert_not_called() + + # Tests for service_tier in response diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index f56910ec103..960f6248e4b 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -9,6 +9,7 @@ from contextlib import AsyncExitStack, closing from datetime import datetime, timezone from io import BytesIO from pathlib import Path +from types import MappingProxyType from typing import Final from unittest.mock import MagicMock from urllib.parse import unquote, urlparse @@ -98,6 +99,234 @@ class TestBedrockFilesTransformation: assert "_litellm_undeclared_sentinel" not in model_input, model_input assert result[0]["modelInput"]["max_tokens"] == 10 + @staticmethod + def _nova_row( + url: str = "/v1/chat/completions", + model: str = "us.amazon.nova-pro-v1:0", + body_params: Mapping[str, object] = MappingProxyType({}), + ) -> Mapping[str, object]: + prompt: Final = {"input": "hi"} if url == "/v1/responses" else {"messages": [{"role": "user", "content": "hi"}]} + return { + "custom_id": "nova-tier-1", + "method": "POST", + "url": url, + "body": {"model": model, **prompt, **body_params}, + } + + @pytest.mark.parametrize("tier", [None, "auto", "AUTO"]) + def test_nova_row_with_null_or_auto_service_tier_sends_no_tier(self, tier: str | None): + result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( + [self._nova_row(body_params={"service_tier": tier})] + ) + + assert "serviceTier" not in result[0]["modelInput"] + + @pytest.mark.parametrize("tier", ["ultrafast", "reserved", "priority", "standard_only"]) + @pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"]) + def test_nova_row_with_a_service_tier_fails_the_row_without_drop_params(self, url: str, tier: str): + with pytest.raises(litellm.UnsupportedParamsError, match=f"service_tier='{tier}'"): + BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( + [self._nova_row(url, body_params={"service_tier": tier})] + ) + + @pytest.mark.parametrize("tier", ["ultrafast", "reserved"]) + @pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"]) + def test_nova_row_level_drop_params_drops_the_service_tier(self, url: str, tier: str): + result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( + [self._nova_row(url, body_params={"service_tier": tier, "drop_params": True})] + ) + + assert "serviceTier" not in result[0]["modelInput"] + assert "drop_params" not in json.dumps(result[0]["modelInput"]) + + def test_nova_row_service_tier_drops_under_global_drop_params(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "drop_params", True) + + result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( + [self._nova_row(body_params={"service_tier": "reserved"})] + ) + + assert "serviceTier" not in result[0]["modelInput"] + + @pytest.mark.parametrize("tier", ["ultrafast", "reserved"]) + def test_deployment_drop_params_drops_the_service_tier_from_the_uploaded_nova_rows(self, tier: str): + request: Final = BedrockFilesConfig().transform_create_file_request( + model="us.amazon.nova-pro-v1:0", + create_file_data={ + "file": ( + "batch.jsonl", + json.dumps(self._nova_row(body_params={"service_tier": tier})).encode(), + "application/jsonl", + ), + "purpose": "batch", + }, + optional_params={ + "aws_access_key_id": "test-key-id", + "aws_secret_access_key": "test-secret", + "aws_region_name": "us-west-2", + }, + litellm_params={"s3_bucket_name": "litellm-batch-bucket", "drop_params": True}, + ) + + assert isinstance(request, dict) + uploaded_row: Final = json.loads(request["data"]) + assert uploaded_row["modelInput"]["messages"] + assert "serviceTier" not in uploaded_row["modelInput"] + + @pytest.mark.parametrize("drop_params, drops", [(False, False), ("false", False), (True, True), ("true", True)]) + def test_deployment_drop_params_decides_whether_a_claude_row_tier_fails_or_drops( + self, drop_params: bool | str, drops: bool + ): + claude_model: Final = "us.anthropic.claude-haiku-4-5-20251001-v1:0" + row: Final = { + "custom_id": "claude-tier-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": claude_model, + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 10, + "service_tier": "ultrafast", + }, + } + + def upload() -> Mapping[str, object]: + request: Final = BedrockFilesConfig().transform_create_file_request( + model=claude_model, + create_file_data={ + "file": ("batch.jsonl", json.dumps(row).encode(), "application/jsonl"), + "purpose": "batch", + }, + optional_params={ + "aws_access_key_id": "test-key-id", + "aws_secret_access_key": "test-secret", + "aws_region_name": "us-west-2", + }, + litellm_params={"s3_bucket_name": "litellm-batch-bucket", "drop_params": drop_params}, + ) + assert isinstance(request, dict) + return json.loads(request["data"]) + + if not drops: + with pytest.raises(BedrockError, match="service_tier"): + upload() + return + + model_input: Final = upload()["modelInput"] + assert model_input["messages"] + assert "service_tier" not in json.dumps(model_input) + assert "serviceTier" not in model_input + + @pytest.mark.parametrize("tier", [None, "auto", "AUTO"]) + def test_claude_row_with_null_or_auto_service_tier_sends_no_tier(self, tier: str | None): + result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( + [ + self._nova_row( + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", + body_params={"service_tier": tier, "max_tokens": 10}, + ) + ] + ) + + assert result[0]["modelInput"]["messages"] + assert "service_tier" not in json.dumps(result[0]["modelInput"]) + + @pytest.mark.parametrize("drop_params, drops", [(True, True), ("true", True), (False, False), ("false", False)]) + @pytest.mark.parametrize( + "model", ["us.anthropic.claude-haiku-4-5-20251001-v1:0", "openai.gpt-oss-120b-1:0", "us.amazon.nova-pro-v1:0"] + ) + def test_row_level_drop_params_decides_whether_a_rejected_tier_fails_or_drops_for_every_row_format( + self, model: str, drop_params: bool | str, drops: bool + ): + rows: Final = [ + self._nova_row( + model=model, body_params={"service_tier": "ultrafast", "drop_params": drop_params, "max_tokens": 10} + ) + ] + + if not drops: + with pytest.raises(litellm.UnsupportedParamsError, match="service_tier='ultrafast'"): + BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(rows) + return + + result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(rows) + assert result[0]["modelInput"]["messages"] + assert "ultrafast" not in json.dumps(result[0]["modelInput"]) + + @pytest.mark.parametrize( + "tier, bedrock_tier", + [("priority", "priority"), ("Flex", "flex"), ("standard", "default"), ("default", "default")], + ) + @pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"]) + def test_openai_format_row_forwards_a_tier_its_invoke_body_takes(self, url: str, tier: str, bedrock_tier: str): + result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( + [self._nova_row(url, model="openai.gpt-oss-120b-1:0", body_params={"service_tier": tier})] + ) + + assert result[0]["modelInput"]["service_tier"] == bedrock_tier + + @pytest.mark.parametrize("tier", [None, "auto", "AUTO"]) + def test_openai_format_row_with_null_or_auto_service_tier_sends_no_tier(self, tier: str | None): + result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( + [self._nova_row(model="openai.gpt-oss-120b-1:0", body_params={"service_tier": tier})] + ) + + assert "service_tier" not in result[0]["modelInput"] + + @pytest.mark.parametrize("tier", ["ultrafast", "reserved", "scale"]) + @pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"]) + def test_openai_format_row_with_a_tier_bedrock_rejects_fails_without_drop_params(self, url: str, tier: str): + with pytest.raises(litellm.UnsupportedParamsError, match=f"service_tier='{tier}'"): + BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( + [self._nova_row(url, model="openai.gpt-oss-120b-1:0", body_params={"service_tier": tier})] + ) + + @pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"]) + def test_passthrough_row_keeps_row_level_drop_params_out_of_the_model_input(self, url: str): + result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content( + [self._nova_row(url, "openai.gpt-oss-120b-1:0", {"drop_params": True})] + ) + + assert "drop_params" not in result[0]["modelInput"] + + @pytest.mark.parametrize( + "row_drop_params, deployment_drop_params, drops", + [(True, False, True), ("true", False, True), (None, True, True), (None, False, False), ("false", False, False)], + ) + def test_drop_params_decides_whether_a_param_nova_rejects_fails_or_drops_from_an_uploaded_nova_row( + self, row_drop_params: bool | str | None, deployment_drop_params: bool, drops: bool + ): + tool: Final = {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}} + row_params: Final = { + "tools": [tool], + "tool_choice": "none", + **({} if row_drop_params is None else {"drop_params": row_drop_params}), + } + row_bytes: Final = json.dumps(self._nova_row(body_params=row_params)).encode() + + def upload() -> Mapping[str, object]: + request: Final = BedrockFilesConfig().transform_create_file_request( + model="us.amazon.nova-pro-v1:0", + create_file_data={"file": ("batch.jsonl", row_bytes, "application/jsonl"), "purpose": "batch"}, + optional_params={ + "aws_access_key_id": "test-key-id", + "aws_secret_access_key": "test-secret", + "aws_region_name": "us-west-2", + }, + litellm_params={"s3_bucket_name": "litellm-batch-bucket", "drop_params": deployment_drop_params}, + ) + assert isinstance(request, dict) + return json.loads(request["data"]) + + if not drops: + with pytest.raises(BedrockError, match="tool_choice"): + upload() + return + + tool_config: Final = upload()["modelInput"]["toolConfig"] + assert tool_config["tools"] + assert "toolChoice" not in tool_config + def test_nova_text_only_uses_converse_format(self): """ Test that Nova models produce Converse API format in batch modelInput.