mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-14 09:03:16 +00:00
fix(bedrock): answer 400 for service tiers Bedrock cannot serve, forward reserved (#50516)
* fix(bedrock): answer 400 for service tiers Bedrock cannot serve, forward reserved Converse used to drop any service_tier outside default, flex, and priority with no error, rewrite auto to default, and drop reserved. Now reserved and standard_only map to Bedrock tiers, auto sends no tier, and any other value answers 400 unless drop_params is on. Nova batch rows honor the row's drop_params. * fix(bedrock): keep a Responses batch row's drop_params and keep it out of passthrough rows * test(bedrock): annotate service tier test locals as Final and drop the row helper's kwargs * fix(bedrock): refuse a service tier on Nova batch rows unless drop_params, and honor the deployment's drop_params * fix(bedrock): map standard and any-case service tiers, and drop tiers from Claude batch rows under the deployment's drop_params * fix(bedrock): forward the tiers OpenAI-format batch rows take, accept auto on Claude rows, and label tiers in lowercase * fix(bedrock): mark the service tier block field read-only * fix(bedrock): read drop_params given as a string on batch rows, and cover the Nova row mapper's drop_params * test(prometheus): sort the service tier label test imports * fix(bedrock): send the Invoke route's service tier as a header and name the accepted tiers in the 400 * fix(bedrock): report and bill the tier Bedrock served on Invoke calls and streams, and send the tier header only to Nova * fix(bedrock): return copies when stamping the served tier, read its header without re-encoding the others, and pin it per stream chunk The Nova Invoke mapper and the event-stream decoder now return model_copy results instead of writing service_tier in place. One shared reader finds the X-Amzn-Bedrock-Service-Tier response header case-insensitively, so a non-ASCII request id no longer breaks the stream decoder, and the passthrough uses the same reader. Tests now vary the served tier in both directions, check every chunk, and cover a stream with no tier header or an empty one * test(bedrock): check the served tier on a stream whose headers come as a plain dict with an upper-case name The non-ASCII request id stream test now also carries X-AMZN-BEDROCK-SERVICE-TIER in a plain dict and asserts every chunk reports that tier, so a case-sensitive header reader or a lookup by the canonical name fails it * refactor(bedrock): drop the passthrough's own service tier header reader that main re-added The passthrough already reads the served tier through the shared case-insensitive reader in common_utils, so the httpx.Headers copy main brought back with the embeddings change has no callers and references a constant this branch removed --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
quietjules
GitHub
parent
03f14051b9
commit
f5ef14308b
15 changed files with 786 additions and 66 deletions
|
|
@ -5,13 +5,14 @@ from typing import Final
|
|||
from litellm.types.utils import ServiceTier, StandardLoggingPayload
|
||||
|
||||
# Tiers a caller may name in a request, across the providers that accept the
|
||||
# parameter: OpenAI ("auto", "default", "flex", "priority", "scale"), Bedrock and
|
||||
# Groq (subsets of those), Anthropic ("auto", "standard_only") and Vertex, which
|
||||
# maps "default" to "standard". Bounds the caller-controlled requested tier
|
||||
# parameter: OpenAI ("auto", "default", "flex", "priority", "scale"), Bedrock
|
||||
# ("auto", "default", "flex", "priority", "reserved", "standard", "standard_only"),
|
||||
# Groq (a subset of OpenAI's), Anthropic ("auto", "standard_only") and Vertex,
|
||||
# which maps "default" to "standard". Bounds the caller-controlled requested tier
|
||||
# wherever it is recorded. Derived from ``ServiceTier`` so a tier added there for
|
||||
# cost calculation cannot go missing here.
|
||||
KNOWN_REQUEST_SERVICE_TIERS: Final = frozenset(
|
||||
tuple(tier.value for tier in ServiceTier) + ("batch", "default", "scale", "standard", "standard_only")
|
||||
tuple(tier.value for tier in ServiceTier) + ("batch", "default", "reserved", "scale", "standard", "standard_only")
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -20,8 +21,7 @@ def get_served_service_tier(standard_logging_payload: StandardLoggingPayload) ->
|
|||
The tier the provider reports it actually served the request on.
|
||||
|
||||
Providers report it either at the top level of the response (OpenAI, Bedrock,
|
||||
Groq) or on the usage object (Anthropic). Streaming responses carry no served
|
||||
tier.
|
||||
Groq) or on the usage object (Anthropic).
|
||||
"""
|
||||
response: Final = standard_logging_payload.get("response")
|
||||
usage_object: Final = standard_logging_payload.get("metadata", {}).get("usage_object")
|
||||
|
|
@ -38,13 +38,14 @@ def get_requested_service_tier(standard_logging_payload: StandardLoggingPayload)
|
|||
The tier the caller asked for, as sent to the provider.
|
||||
|
||||
The value is caller-controlled and survives param mapping even where the
|
||||
provider then ignores it (Bedrock and Groq accept the request and drop an
|
||||
unrecognized tier), so it is only reported when it names a known tier.
|
||||
provider then drops it (Groq for an unrecognized tier, Bedrock for one it
|
||||
cannot serve under ``drop_params``), so it is only reported when it names a
|
||||
known tier, in lowercase since Bedrock takes any case.
|
||||
"""
|
||||
model_parameters: Final = standard_logging_payload.get("model_parameters")
|
||||
requested_tier: Final = model_parameters.get("service_tier") if isinstance(model_parameters, dict) else None
|
||||
if isinstance(requested_tier, str) and requested_tier in KNOWN_REQUEST_SERVICE_TIERS:
|
||||
return requested_tier
|
||||
if isinstance(requested_tier, str) and requested_tier.lower() in KNOWN_REQUEST_SERVICE_TIERS:
|
||||
return requested_tier.lower()
|
||||
return None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -202,9 +202,16 @@ def _tools_the_model_accepts(
|
|||
|
||||
_OUTPUT_CONFIG_BLOCK: Final = TypeAdapter(OutputConfigBlock)
|
||||
_DEEPSEEK_THINKING_REASON: Final = "DeepSeek on Bedrock reasons natively and takes no thinking field."
|
||||
BEDROCK_SERVICE_TIERS: Final = ("default", "flex", "priority", "reserved")
|
||||
_BEDROCK_SERVICE_TIER_ALIASES: Final = types.MappingProxyType({"standard": "default", "standard_only": "default"})
|
||||
_DEEPSEEK_R1_EFFORT_REASON: Final = "DeepSeek R1 on Bedrock always reasons and rejects reasoning_effort."
|
||||
|
||||
|
||||
def bedrock_service_tier_name(requested_tier: str) -> str:
|
||||
tier: Final = requested_tier.lower()
|
||||
return _BEDROCK_SERVICE_TIER_ALIASES.get(tier, tier)
|
||||
|
||||
|
||||
def _has_native_text_format(output_config: object) -> bool:
|
||||
if not isinstance(output_config, Mapping):
|
||||
return False
|
||||
|
|
@ -1253,8 +1260,13 @@ class AmazonConverseConfig(BaseConfig):
|
|||
self._map_context_management_param(value, optional_params)
|
||||
if param == "requestMetadata":
|
||||
self._map_request_metadata_param(value, optional_params, drop_params)
|
||||
if param == "service_tier" and isinstance(value, str):
|
||||
self._map_service_tier_param(value, optional_params)
|
||||
if param == "service_tier":
|
||||
self._map_service_tier_param(
|
||||
REQUEST_PARAMS.validate_python(request_params)["service_tier"],
|
||||
optional_params,
|
||||
model,
|
||||
drop_params,
|
||||
)
|
||||
|
||||
if param == "web_search_options" and isinstance(value, dict):
|
||||
# Note: we use `isinstance(value, dict)` instead of `value and isinstance(value, dict)`
|
||||
|
|
@ -1308,17 +1320,25 @@ class AmazonConverseConfig(BaseConfig):
|
|||
if mapped is not None:
|
||||
optional_params["context_management"] = mapped
|
||||
|
||||
def _map_service_tier_param(self, value: str, optional_params: dict) -> None:
|
||||
"""Map OpenAI service_tier (string) to Bedrock serviceTier (object).
|
||||
|
||||
OpenAI values: "auto", "default", "flex", "priority"
|
||||
Bedrock values: "default", "flex", "priority" (no "auto")
|
||||
"""
|
||||
bedrock_tier = value
|
||||
if value == "auto":
|
||||
bedrock_tier = "default" # Bedrock doesn't support "auto"
|
||||
if bedrock_tier in ("default", "flex", "priority"):
|
||||
optional_params["serviceTier"] = {"type": bedrock_tier}
|
||||
def _map_service_tier_param(self, value: object, optional_params: dict, model: str, drop_params: bool) -> None:
|
||||
tier: Final = bedrock_service_tier_name(value) if isinstance(value, str) else None
|
||||
if value is None or tier == "auto":
|
||||
return
|
||||
if tier in BEDROCK_SERVICE_TIERS:
|
||||
optional_params["serviceTier"] = {"type": tier}
|
||||
return
|
||||
if not (litellm.drop_params or drop_params):
|
||||
raise litellm.utils.UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} does not support service_tier={value!r} on this Bedrock route, which takes "
|
||||
f"auto, standard, or one of {', '.join(BEDROCK_SERVICE_TIERS)}. "
|
||||
"To drop unsupported params, set `litellm.drop_params = True`."
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
verbose_logger.warning(
|
||||
"Dropping service_tier=%r for %s, since this Bedrock route does not serve it.", value, model
|
||||
)
|
||||
|
||||
def _translate_response_format_param(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import json
|
||||
import types
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from typing import TYPE_CHECKING, Final
|
||||
from typing import TYPE_CHECKING, Final, TypeVar
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
|
@ -61,6 +61,7 @@ from ..common_utils import (
|
|||
get_bedrock_stream_event_statuses,
|
||||
get_bedrock_tool_name,
|
||||
parse_bedrock_event_stream,
|
||||
served_service_tier,
|
||||
)
|
||||
|
||||
bedrock_tool_name_mappings: Final[InMemoryCache] = InMemoryCache(max_size_in_memory=50, default_ttl=600)
|
||||
|
|
@ -218,6 +219,15 @@ def _stream_decoder(
|
|||
return AWSEventStreamDecoder(model=model, json_mode=json_mode)
|
||||
|
||||
|
||||
_StreamChunk: Final = TypeVar("_StreamChunk")
|
||||
|
||||
|
||||
def _with_served_service_tier(chunk: _StreamChunk, served_tier: str | None) -> _StreamChunk:
|
||||
if served_tier is None or not isinstance(chunk, ModelResponseStream):
|
||||
return chunk
|
||||
return chunk.model_copy(update={"service_tier": served_tier})
|
||||
|
||||
|
||||
async def make_call(
|
||||
client: AsyncHTTPHandler | None,
|
||||
api_base: str,
|
||||
|
|
@ -851,6 +861,7 @@ class AWSEventStreamDecoder:
|
|||
event_stream_buffer: Final = EventStreamBuffer()
|
||||
tally: Final = _EventStreamTally(get_bedrock_stream_event_statuses())
|
||||
request_id_headers: Final = provider_request_id_headers(response_headers)
|
||||
served_tier: Final = served_service_tier(response_headers)
|
||||
for chunk in iterator:
|
||||
event_stream_buffer.add_data(chunk)
|
||||
tally.add_chunk(chunk)
|
||||
|
|
@ -859,7 +870,7 @@ class AWSEventStreamDecoder:
|
|||
if message:
|
||||
# sse_event = ServerSentEvent(data=message, event="completion")
|
||||
_data = json.loads(message)
|
||||
yield self.chunk_parser(chunk_data=_data)
|
||||
yield _with_served_service_tier(self.chunk_parser(chunk_data=_data), served_tier)
|
||||
undecoded_stream_error: Final = tally.undecoded_stream_error(response_headers)
|
||||
if undecoded_stream_error is not None:
|
||||
raise undecoded_stream_error
|
||||
|
|
@ -876,6 +887,7 @@ class AWSEventStreamDecoder:
|
|||
event_stream_buffer: Final = EventStreamBuffer()
|
||||
tally: Final = _EventStreamTally(get_bedrock_stream_event_statuses())
|
||||
request_id_headers: Final = provider_request_id_headers(response_headers)
|
||||
served_tier: Final = served_service_tier(response_headers)
|
||||
async for chunk in iterator:
|
||||
event_stream_buffer.add_data(chunk)
|
||||
tally.add_chunk(chunk)
|
||||
|
|
@ -883,7 +895,7 @@ class AWSEventStreamDecoder:
|
|||
message = self._decode_event(event, tally, request_id_headers)
|
||||
if message:
|
||||
_data = json.loads(message)
|
||||
yield self.chunk_parser(chunk_data=_data)
|
||||
yield _with_served_service_tier(self.chunk_parser(chunk_data=_data), served_tier)
|
||||
undecoded_stream_error: Final = tally.undecoded_stream_error(response_headers)
|
||||
if undecoded_stream_error is not None:
|
||||
raise undecoded_stream_error
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from litellm.types.llms.bedrock import (
|
|||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
from ...common_utils import served_service_tier
|
||||
from ..converse_transformation import AmazonConverseConfig
|
||||
from .base_invoke_transformation import AmazonInvokeConfig
|
||||
|
||||
|
|
@ -142,7 +143,7 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig):
|
|||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
return AmazonConverseConfig.transform_response(
|
||||
response: Final = AmazonConverseConfig.transform_response(
|
||||
self,
|
||||
model,
|
||||
raw_response,
|
||||
|
|
@ -156,6 +157,10 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig):
|
|||
api_key,
|
||||
json_mode,
|
||||
)
|
||||
served_tier: Final = served_service_tier(raw_response.headers)
|
||||
if served_tier is None:
|
||||
return response
|
||||
return response.model_copy(update={"service_tier": served_tier})
|
||||
|
||||
@staticmethod
|
||||
def _inline_cache_points(request: BedrockInvokeNovaRequest) -> BedrockInvokeNovaRequest:
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
|
|||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
from litellm.llms.bedrock.chat.invoke_handler import make_call, make_sync_call
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.llms.bedrock.common_utils import BEDROCK_SERVICE_TIER_HEADER, BedrockError
|
||||
from litellm.llms.bedrock.request_metadata import (
|
||||
bedrock_request_metadata_headers,
|
||||
merge_bedrock_invoke_headers,
|
||||
|
|
@ -67,8 +67,13 @@ class _LlamaInvokeResponse(TypedDict):
|
|||
generation: ReadOnly[str | None]
|
||||
|
||||
|
||||
class _InvokeServiceTierParams(TypedDict):
|
||||
serviceTier: NotRequired[ReadOnly[object]]
|
||||
|
||||
|
||||
_COHERE_INVOKE_RESPONSE: Final = TypeAdapter(_CohereInvokeResponse)
|
||||
_LLAMA_INVOKE_RESPONSE: Final = TypeAdapter(_LlamaInvokeResponse)
|
||||
_INVOKE_SERVICE_TIER_PARAMS: Final = TypeAdapter(_InvokeServiceTierParams)
|
||||
|
||||
_GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
|
||||
"{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
|
||||
|
|
@ -79,6 +84,14 @@ def _response_header(headers: Mapping[str, str], name: str) -> str | None:
|
|||
return headers.get(name, None)
|
||||
|
||||
|
||||
def _bedrock_invoke_service_tier_headers(optional_params: _InvokeServiceTierParams) -> tuple[tuple[str, str], ...]:
|
||||
match optional_params.get("serviceTier"):
|
||||
case {"type": str(tier)}:
|
||||
return ((BEDROCK_SERVICE_TIER_HEADER, tier),)
|
||||
case _:
|
||||
return ()
|
||||
|
||||
|
||||
def bedrock_invoke_guardrail_headers(raw_guardrail_config: object) -> "dict[str, str]":
|
||||
try:
|
||||
guardrail_config: Final = _GUARDRAIL_CONFIG_VALIDATOR.validate_python(raw_guardrail_config)
|
||||
|
|
@ -455,8 +468,15 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
if raw_guardrail_config is None
|
||||
else tuple(bedrock_invoke_guardrail_headers(raw_guardrail_config).items())
|
||||
)
|
||||
service_tier_headers: Final = (
|
||||
_bedrock_invoke_service_tier_headers(_INVOKE_SERVICE_TIER_PARAMS.validate_python(optional_params))
|
||||
if self.get_bedrock_invoke_provider(model) == "nova"
|
||||
else ()
|
||||
)
|
||||
owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params)
|
||||
return merge_bedrock_invoke_headers(headers, guardrail_headers, metadata_headers, owned_names)
|
||||
return merge_bedrock_invoke_headers(
|
||||
headers, (*guardrail_headers, *service_tier_headers), metadata_headers, owned_names
|
||||
)
|
||||
|
||||
def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ if TYPE_CHECKING:
|
|||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
BEDROCK_INVOKE_TRACE_FIELD: Final = "amazon-bedrock-trace"
|
||||
BEDROCK_SERVICE_TIER_HEADER: Final = "X-Amzn-Bedrock-Service-Tier"
|
||||
BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
"compact_20260112": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
|
||||
|
|
@ -86,6 +87,13 @@ BedrockRoute = Literal[
|
|||
]
|
||||
|
||||
|
||||
def served_service_tier(response_headers: Mapping[str, str] | None) -> str | None:
|
||||
tier_header: Final = BEDROCK_SERVICE_TIER_HEADER.lower()
|
||||
return next(
|
||||
(value for name, value in (response_headers or {}).items() if name.lower() == tier_header and value), None
|
||||
)
|
||||
|
||||
|
||||
def error_response_text(response: httpx.Response) -> str:
|
||||
try:
|
||||
return response.text
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ from typing_extensions import TypedDict as ExtraItemsTypedDict
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.constants import BEDROCK_INVOKE_PROVIDERS_LITERAL
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.exceptions import BadRequestError, UnsupportedParamsError
|
||||
from litellm.files.utils import FilesAPIUtils
|
||||
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
|
||||
from litellm.litellm_core_utils.cloud_storage_security import (
|
||||
|
|
@ -40,7 +40,7 @@ from litellm.litellm_core_utils.cloud_storage_security import (
|
|||
split_configured_cloud_bucket_name,
|
||||
validate_managed_cloud_file_id,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import require_attributes
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params, require_attributes
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
text_completion_prompt_to_messages,
|
||||
)
|
||||
|
|
@ -49,7 +49,7 @@ from litellm.litellm_core_utils.prompt_templates.file_id_utils import (
|
|||
)
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError
|
||||
from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS, BaseLLMException
|
||||
from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body
|
||||
from litellm.llms.base_llm.files.transformation import (
|
||||
BaseFilesConfig,
|
||||
|
|
@ -391,6 +391,42 @@ def _batch_line_error(line_number: int, problem: str) -> BedrockError:
|
|||
return _client_error(f"Batch input file line {line_number} {problem}")
|
||||
|
||||
|
||||
_OPENAI_FORMAT_INVOKE_SERVICE_TIERS: Final = frozenset({"default", "flex", "priority"})
|
||||
|
||||
|
||||
def _check_batch_record_service_tier(
|
||||
value: object, model: str, drop_params: bool, accepted_tiers: frozenset[str] = frozenset()
|
||||
) -> None:
|
||||
import litellm
|
||||
|
||||
if value is None or (isinstance(value, str) and value.lower() == "auto"):
|
||||
return
|
||||
if not (litellm.drop_params or drop_params):
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} batch records do not support service_tier={value!r}. "
|
||||
+ (
|
||||
f"Their InvokeModel body takes only {', '.join(sorted(accepted_tiers))}. "
|
||||
if accepted_tiers
|
||||
else "A Bedrock batch record for this model is an InvokeModel body, which takes no service tier. "
|
||||
)
|
||||
+ "To drop unsupported params, set `litellm.drop_params = True`."
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
verbose_logger.warning("Dropping service_tier=%r from a %s batch record, since Bedrock rejects it.", value, model)
|
||||
|
||||
|
||||
def _openai_format_batch_record_service_tier(value: object, model: str, drop_params: bool) -> Mapping[str, str]:
|
||||
from litellm.llms.bedrock.chat.converse_transformation import bedrock_service_tier_name
|
||||
|
||||
tier: Final = bedrock_service_tier_name(value) if isinstance(value, str) else None
|
||||
if tier in _OPENAI_FORMAT_INVOKE_SERVICE_TIERS:
|
||||
return MappingProxyType({"service_tier": tier})
|
||||
_check_batch_record_service_tier(value, model, drop_params, _OPENAI_FORMAT_INVOKE_SERVICE_TIERS)
|
||||
return MappingProxyType({})
|
||||
|
||||
|
||||
def _decode_batch_input(content: str | bytes) -> str:
|
||||
if isinstance(content, str):
|
||||
return content.removeprefix("\ufeff")
|
||||
|
|
@ -1047,7 +1083,11 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
|||
model), so `input`, `instructions`, `max_output_tokens` and the tool
|
||||
params translate identically in batch and real time.
|
||||
"""
|
||||
return responses_batch_body_to_chat_body(openai_request_body)
|
||||
chat_body: Final = responses_batch_body_to_chat_body(openai_request_body)
|
||||
drop_params: Final = openai_request_body.get("drop_params")
|
||||
if drop_params is None:
|
||||
return chat_body
|
||||
return _frozen_mapping(chain(chat_body.items(), (("drop_params", drop_params),)))
|
||||
|
||||
@staticmethod
|
||||
def _transform_batch_body_to_chat_body(
|
||||
|
|
@ -1073,6 +1113,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
|||
openai_request_body: Mapping[str, Any],
|
||||
model: str,
|
||||
provider: str | None = None,
|
||||
drop_params: bool = False,
|
||||
model_info: Mapping[str, object] | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
|
|
@ -1088,6 +1129,10 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
|||
messages: Final = map_developer_role_to_system_role(openai_request_body.get("messages", []))
|
||||
request_params: Final[Mapping[str, object]] = openai_request_body
|
||||
optional_params: Final = {k: v for k, v in request_params.items() if k not in ["model", "messages"]}
|
||||
drop_unsupported_params: Final = (
|
||||
drop_params or normalize_drop_params(optional_params.get("drop_params")) is True
|
||||
)
|
||||
requested_service_tier: Final = REQUEST_PARAMS.validate_python(optional_params).get("service_tier")
|
||||
|
||||
# --- Anthropic: use existing AmazonAnthropicClaudeConfig ---
|
||||
if provider == LlmProviders.ANTHROPIC:
|
||||
|
|
@ -1095,6 +1140,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
|||
AmazonAnthropicClaudeConfig,
|
||||
)
|
||||
|
||||
_check_batch_record_service_tier(requested_service_tier, model, drop_unsupported_params)
|
||||
config: Final = AmazonAnthropicClaudeConfig()
|
||||
litellm_params: Final = without_bedrock_native_output_opt_in(
|
||||
{
|
||||
|
|
@ -1114,8 +1160,10 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
|||
**{
|
||||
k: v
|
||||
for k, v in optional_params.items()
|
||||
if not is_litellm_owned_kwarg(k) or k in _LITELLM_PARAMS_THE_MAPPER_TAKES
|
||||
if k != "service_tier"
|
||||
and (not is_litellm_owned_kwarg(k) or k in _LITELLM_PARAMS_THE_MAPPER_TAKES)
|
||||
}
|
||||
| ({"drop_params": True} if drop_params else {})
|
||||
),
|
||||
)
|
||||
return config.transform_request(
|
||||
|
|
@ -1136,12 +1184,15 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
|||
AmazonConverseConfig,
|
||||
)
|
||||
|
||||
_check_batch_record_service_tier(requested_service_tier, model, drop_unsupported_params)
|
||||
converse_config: Final = AmazonConverseConfig()
|
||||
mapped_params = converse_config.map_openai_params(
|
||||
non_default_params={k: v for k, v in optional_params.items() if k != "requestMetadata"},
|
||||
non_default_params={
|
||||
k: v for k, v in optional_params.items() if k not in ("requestMetadata", "service_tier")
|
||||
},
|
||||
optional_params={},
|
||||
model=model,
|
||||
drop_params=False,
|
||||
drop_params=drop_unsupported_params,
|
||||
)
|
||||
return converse_config.transform_request(
|
||||
model=model,
|
||||
|
|
@ -1155,7 +1206,8 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
|||
# like openai.gpt-oss-*, qwen, deepseek, etc.) ---
|
||||
return {
|
||||
"messages": messages,
|
||||
**optional_params,
|
||||
**{k: v for k, v in optional_params.items() if k not in ("drop_params", "service_tier")},
|
||||
**_openai_format_batch_record_service_tier(requested_service_tier, model, drop_unsupported_params),
|
||||
}
|
||||
|
||||
def _resolve_batch_record_model_and_provider(
|
||||
|
|
@ -1203,17 +1255,25 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
|||
batch_input_lines: Sequence[_BatchInputLine],
|
||||
target_model: str = "",
|
||||
model_info: Mapping[str, object] | None = None,
|
||||
drop_params: bool = False,
|
||||
) -> tuple[_BedrockBatchRecord, ...]:
|
||||
return tuple(
|
||||
self._transform_batch_input_line(idx, batch_input_line, target_model, model_info)
|
||||
self._transform_batch_input_line(idx, batch_input_line, target_model, model_info, drop_params)
|
||||
for idx, batch_input_line in enumerate(batch_input_lines)
|
||||
)
|
||||
|
||||
def _transform_batch_input_line(
|
||||
self, idx: int, batch_input_line: _BatchInputLine, target_model: str, model_info: Mapping[str, object] | None
|
||||
self,
|
||||
idx: int,
|
||||
batch_input_line: _BatchInputLine,
|
||||
target_model: str,
|
||||
model_info: Mapping[str, object] | None,
|
||||
drop_params: bool,
|
||||
) -> _BedrockBatchRecord:
|
||||
try:
|
||||
return self._transform_openai_batch_record(idx, batch_input_line.record, target_model, model_info)
|
||||
return self._transform_openai_batch_record(
|
||||
idx, batch_input_line.record, target_model, model_info, drop_params
|
||||
)
|
||||
except _BATCH_RECORD_CONVERSION_ERRORS as e:
|
||||
raise _batch_line_error(
|
||||
batch_input_line.line_number, f"could not be converted to a Bedrock batch record: {e}"
|
||||
|
|
@ -1225,6 +1285,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
|||
openai_record: _OpenAIBatchRecord,
|
||||
target_model: str,
|
||||
model_info: Mapping[str, object] | None = None,
|
||||
drop_params: bool = False,
|
||||
) -> _BedrockBatchRecord:
|
||||
import litellm
|
||||
|
||||
|
|
@ -1249,6 +1310,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
|||
model=model_for_transform,
|
||||
provider=provider,
|
||||
model_info=model_info,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
)
|
||||
return _BedrockBatchRecord(
|
||||
|
|
@ -1293,8 +1355,12 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
|||
batch_input_lines: Final = _parse_batch_jsonl(original_file_content)
|
||||
litellm_params_model: Final = litellm_params.get("model")
|
||||
target_model: Final = model or (litellm_params_model if isinstance(litellm_params_model, str) else "")
|
||||
deployment_drop_params: Final = REQUEST_PARAMS.validate_python(litellm_params).get("drop_params")
|
||||
bedrock_jsonl_content: Final = self.transform_batch_input_lines(
|
||||
batch_input_lines, target_model=target_model, model_info=_upload_deployment_model_info(litellm_params)
|
||||
batch_input_lines,
|
||||
target_model=target_model,
|
||||
model_info=_upload_deployment_model_info(litellm_params),
|
||||
drop_params=normalize_drop_params(deployment_drop_params) is True,
|
||||
)
|
||||
file_content = "\n".join(json.dumps(item) for item in bedrock_jsonl_content)
|
||||
elif isinstance(extracted_file_data_content, bytes):
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ from ..common_utils import (
|
|||
BedrockError,
|
||||
BedrockEventStreamDecoderBase,
|
||||
BedrockModelInfo,
|
||||
served_service_tier,
|
||||
)
|
||||
from ..request_metadata import (
|
||||
BEDROCK_REQUEST_METADATA_HEADER,
|
||||
|
|
@ -59,7 +60,6 @@ _BEDROCK_INVOKE_ACTIONS: Final = frozenset({"invoke", "invoke-with-response-stre
|
|||
_BEDROCK_INVOKE_GUARDRAIL_HEADERS: Final = frozenset(
|
||||
{"x-amzn-bedrock-guardrailidentifier", "x-amzn-bedrock-guardrailversion", "x-amzn-bedrock-trace"}
|
||||
)
|
||||
_BEDROCK_SERVICE_TIER_HEADER: Final = "x-amzn-bedrock-service-tier"
|
||||
_NO_RESPONSE_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
|
||||
_TieredResponse: Final = TypeVar("_TieredResponse", bound=BaseModel)
|
||||
|
||||
|
|
@ -423,13 +423,6 @@ def _invoke_embedding_response(model: str, httpx_response: Response, has_input_i
|
|||
)
|
||||
|
||||
|
||||
def _served_service_tier(response_headers: Mapping[str, str]) -> str | None:
|
||||
headers: Final = httpx.Headers(response_headers)
|
||||
if _BEDROCK_SERVICE_TIER_HEADER not in headers:
|
||||
return None
|
||||
return headers[_BEDROCK_SERVICE_TIER_HEADER]
|
||||
|
||||
|
||||
def _with_served_service_tier(response: _TieredResponse, service_tier: str | None) -> _TieredResponse:
|
||||
if service_tier is None:
|
||||
return response
|
||||
|
|
@ -659,7 +652,7 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD
|
|||
if "invoke" in endpoint:
|
||||
logging_obj.record_native_bedrock_guardrail(_invoke_guardrail_trace(httpx_response))
|
||||
|
||||
return _with_served_service_tier(litellm_model_response, _served_service_tier(httpx_response.headers))
|
||||
return _with_served_service_tier(litellm_model_response, served_service_tier(httpx_response.headers))
|
||||
|
||||
def create_stream_collector(
|
||||
self,
|
||||
|
|
@ -672,7 +665,7 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD
|
|||
return BedrockEventStreamCollector(
|
||||
parse_event=self._parse_message_from_event,
|
||||
decoder=self._get_event_stream_decoder(model=model, endpoint=endpoint),
|
||||
service_tier=_served_service_tier(response_headers),
|
||||
service_tier=served_service_tier(response_headers),
|
||||
)
|
||||
|
||||
def _get_event_stream_decoder(self, model: str, endpoint: str) -> Optional["AWSEventStreamDecoder"]:
|
||||
|
|
|
|||
|
|
@ -244,7 +244,7 @@ class ConverseTokenUsageBlock(TypedDict, total=False):
|
|||
|
||||
|
||||
class ServiceTierBlock(TypedDict):
|
||||
type: Literal["priority", "default", "flex"]
|
||||
type: ReadOnly[Literal["priority", "default", "flex", "reserved"]]
|
||||
|
||||
|
||||
class ConverseResponseBlock(TypedDict, total=False):
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ Run with:
|
|||
"""
|
||||
|
||||
import datetime
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -18,6 +19,7 @@ from litellm.litellm_core_utils.service_tier_utils import (
|
|||
KNOWN_REQUEST_SERVICE_TIERS,
|
||||
get_service_tier_from_standard_logging_payload,
|
||||
)
|
||||
from litellm.llms.bedrock.chat.converse_transformation import BEDROCK_SERVICE_TIERS
|
||||
from litellm.types.integrations.prometheus import (
|
||||
PrometheusMetricLabels,
|
||||
UserAPIKeyLabelNames,
|
||||
|
|
@ -138,7 +140,7 @@ def test_requested_tier_used_when_no_served_tier():
|
|||
def test_unrecognized_requested_tier_is_not_labelled():
|
||||
"""
|
||||
A caller-supplied tier survives param mapping even where the provider then
|
||||
ignores it (Bedrock and Groq drop an unrecognized tier and still answer), so
|
||||
ignores it (Groq drops an unrecognized tier and still answers), so
|
||||
labelling it verbatim would let one caller mint a series per string.
|
||||
"""
|
||||
payload = _standard_logging_payload(model_parameters={"service_tier": "attacker-controlled-a1b2c3"})
|
||||
|
|
@ -166,6 +168,20 @@ def test_every_known_requested_tier_is_labelled(tier):
|
|||
assert get_service_tier_from_standard_logging_payload(payload) == tier
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tier", BEDROCK_SERVICE_TIERS)
|
||||
def test_every_tier_bedrock_forwards_is_labelled(tier: str):
|
||||
payload: Final = _standard_logging_payload(model_parameters={"service_tier": tier})
|
||||
|
||||
assert get_service_tier_from_standard_logging_payload(payload) == tier
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tier", ["Priority", "FLEX", "Reserved"])
|
||||
def test_a_requested_tier_in_any_case_is_labelled_in_lowercase(tier: str):
|
||||
payload: Final = _standard_logging_payload(model_parameters={"service_tier": tier})
|
||||
|
||||
assert get_service_tier_from_standard_logging_payload(payload) == tier.lower()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"response, usage_object, model_parameters",
|
||||
[
|
||||
|
|
|
|||
|
|
@ -1,11 +1,16 @@
|
|||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.bedrock.chat.invoke_transformations.amazon_nova_transformation import (
|
||||
AmazonInvokeNovaConfig,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.types.integrations.anthropic_cache_control_hook import GATEWAY_INJECTED_CACHE_METADATA_KEY
|
||||
|
||||
MODEL = "us.amazon.nova-pro-v1:0"
|
||||
|
|
@ -134,3 +139,63 @@ def test_caller_request_metadata_is_ignored_because_invoke_model_has_no_field_fo
|
|||
|
||||
assert "requestMetadata" not in request
|
||||
assert request["inferenceConfig"] == {"maxTokens": 8}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("service_tier", "sent_tier"),
|
||||
[("priority", "priority"), ("flex", "flex"), ("reserved", "reserved"), ("Standard", "default"), ("auto", None)],
|
||||
)
|
||||
def test_invoke_sends_the_service_tier_in_the_header_invoke_model_takes(service_tier: str, sent_tier: str | None):
|
||||
send: Final = MagicMock(return_value=httpx.Response(200))
|
||||
client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(send)))
|
||||
|
||||
litellm.completion(
|
||||
model=f"bedrock/invoke/{MODEL}",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
service_tier=service_tier,
|
||||
client=client,
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
|
||||
request: Final = send.call_args.args[0]
|
||||
assert request.headers.get("X-Amzn-Bedrock-Service-Tier") == sent_tier
|
||||
assert "serviceTier" not in json.loads(request.content)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("requested_tier", "served_tier"), [("priority", "flex"), ("flex", "priority")])
|
||||
def test_invoke_reports_and_bills_the_tier_bedrock_served_over_the_tier_requested(
|
||||
requested_tier: str, served_tier: str
|
||||
) -> None:
|
||||
reply: Final = {
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": "hi"}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 1000, "outputTokens": 1000, "totalTokens": 2000},
|
||||
}
|
||||
|
||||
def invoke(tier: Mapping[str, object]) -> litellm.ModelResponse:
|
||||
send: Final = MagicMock(
|
||||
return_value=httpx.Response(200, json=reply, headers={"X-Amzn-Bedrock-Service-Tier": served_tier})
|
||||
)
|
||||
return litellm.completion(
|
||||
model=f"bedrock/invoke/{MODEL}",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(send))),
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-east-1",
|
||||
**tier,
|
||||
)
|
||||
|
||||
native_requested: Final = invoke({"serviceTier": {"type": requested_tier}})
|
||||
openai_requested: Final = invoke({"service_tier": requested_tier})
|
||||
openai_served: Final = invoke({"service_tier": served_tier})
|
||||
|
||||
assert native_requested.service_tier == openai_requested.service_tier == served_tier
|
||||
assert (
|
||||
native_requested._hidden_params["response_cost"]
|
||||
== openai_requested._hidden_params["response_cost"]
|
||||
== openai_served._hidden_params["response_cost"]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -654,3 +654,30 @@ def test_guarded_non_claude_invoke_stream_yields_only_the_text_and_stop_chunks(
|
|||
([(choice.delta.content, choice.finish_reason) for choice in chunk.choices], chunk.provider_specific_fields or {})
|
||||
for chunk in stream
|
||||
] == [([("hi", None)], {}), ([(None, "stop")], {})]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"anthropic.claude-sonnet-4-6",
|
||||
"amazon.titan-text-express-v1",
|
||||
"mistral.mistral-7b-instruct-v0:2",
|
||||
],
|
||||
)
|
||||
def test_a_native_service_tier_sends_no_tier_header_to_invoke_models_whose_replies_do_not_report_it(model: str) -> None:
|
||||
send: Final = MagicMock(return_value=httpx.Response(200))
|
||||
client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(send)))
|
||||
|
||||
litellm.completion(
|
||||
model=f"bedrock/invoke/{model}",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
max_tokens=10,
|
||||
client=client,
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-east-1",
|
||||
serviceTier={"type": "priority"},
|
||||
)
|
||||
|
||||
assert "X-Amzn-Bedrock-Service-Tier" not in send.call_args.args[0].headers
|
||||
|
|
|
|||
|
|
@ -1539,16 +1539,23 @@ async def test_aiter_bytes_stream_error_carries_only_the_provider_request_id_hea
|
|||
|
||||
|
||||
@pytest.mark.parametrize("request_id", ("req-\u00fc-1", "req-\udcfc-1"), ids=("latin-1", "surrogate-escaped"))
|
||||
def test_iter_bytes_streams_and_fails_normally_with_a_non_ascii_provider_request_id(request_id: str) -> None:
|
||||
def test_iter_bytes_streams_the_served_tier_and_fails_normally_with_a_non_ascii_provider_request_id(
|
||||
request_id: str,
|
||||
) -> None:
|
||||
decoder: Final = AWSEventStreamDecoder(model="us.moonshotai.kimi-k3")
|
||||
headers: Final = {"content-type": "application/vnd.amazon.eventstream", "x-amzn-RequestId": request_id}
|
||||
headers: Final = {
|
||||
"content-type": "application/vnd.amazon.eventstream",
|
||||
"x-amzn-RequestId": request_id,
|
||||
"X-AMZN-BEDROCK-SERVICE-TIER": "flex",
|
||||
}
|
||||
|
||||
chunks: Final = list(decoder.iter_bytes(iter(_CONVERSE_CONTENT_FRAMES), response_headers=headers))
|
||||
with pytest.raises(BedrockError) as exc_info:
|
||||
list(decoder.iter_bytes(iter([_unknown_event_frame()]), response_headers=headers))
|
||||
|
||||
texts: Final = [chunk.choices[0].delta.content for chunk in chunks if isinstance(chunk, ModelResponseStream)]
|
||||
assert "".join(text or "" for text in texts) == "hi"
|
||||
stream_chunks: Final = [chunk for chunk in chunks if isinstance(chunk, ModelResponseStream)]
|
||||
assert "".join(chunk.choices[0].delta.content or "" for chunk in stream_chunks) == "hi"
|
||||
assert {chunk.service_tier for chunk in stream_chunks} == {"flex"}
|
||||
assert exc_info.value.status_code == 502
|
||||
assert exc_info.value.headers is None
|
||||
|
||||
|
|
@ -2084,3 +2091,110 @@ def test_invoke_stream_decoder_carries_the_guardrail_trace_of_the_message_stop_c
|
|||
assert (plain.provider_specific_fields or {}).get("amazon-bedrock-trace") is None
|
||||
assert traced.provider_specific_fields is not None
|
||||
assert traced.provider_specific_fields["amazon-bedrock-trace"] == trace
|
||||
|
||||
|
||||
NOVA_INVOKE_STREAM_EVENTS: Final = (
|
||||
{"messageStart": {"role": "assistant"}},
|
||||
{"contentBlockDelta": {"contentBlockIndex": 0, "delta": {"text": "hi"}}},
|
||||
{"contentBlockStop": {"contentBlockIndex": 0}},
|
||||
{"messageStop": {"stopReason": "end_turn"}},
|
||||
{"metadata": {"usage": {"inputTokens": 1000, "outputTokens": 1000, "totalTokens": 2000}, "metrics": {}}},
|
||||
)
|
||||
|
||||
|
||||
def _nova_invoke(served_tier: str | None) -> httpx.MockTransport:
|
||||
tier_header: Final = {} if served_tier is None else {"X-Amzn-Bedrock-Service-Tier": served_tier}
|
||||
|
||||
def reply(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/invoke-with-response-stream"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "application/vnd.amazon.eventstream", **tier_header},
|
||||
content=b"".join(_bedrock_event_stream_frame(event) for event in NOVA_INVOKE_STREAM_EVENTS),
|
||||
)
|
||||
return httpx.Response(
|
||||
200,
|
||||
headers=tier_header,
|
||||
json={
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": "hi"}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 1000, "outputTokens": 1000, "totalTokens": 2000},
|
||||
},
|
||||
)
|
||||
|
||||
return httpx.MockTransport(reply)
|
||||
|
||||
|
||||
async def _nova_invoke_calls(
|
||||
requested_tier: Mapping[str, object], served_tier: str | None
|
||||
) -> tuple[litellm.ModelResponse, Sequence[ModelResponseStream], Sequence[ModelResponseStream]]:
|
||||
def completion(stream: bool) -> object:
|
||||
return litellm.completion(
|
||||
model="bedrock/invoke/us.amazon.nova-pro-v1:0",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=stream,
|
||||
client=HTTPHandler(client=httpx.Client(transport=_nova_invoke(served_tier))),
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-east-1",
|
||||
**requested_tier,
|
||||
)
|
||||
|
||||
async_stream: Final = await litellm.acompletion(
|
||||
model="bedrock/invoke/us.amazon.nova-pro-v1:0",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
client=AsyncHTTPHandler(transport=_nova_invoke(served_tier)),
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-east-1",
|
||||
**requested_tier,
|
||||
)
|
||||
non_streaming: Final = completion(stream=False)
|
||||
assert isinstance(non_streaming, litellm.ModelResponse)
|
||||
return non_streaming, tuple(completion(stream=True)), tuple([chunk async for chunk in async_stream])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("requested_tier", "served_tier"),
|
||||
[
|
||||
pytest.param({"serviceTier": {"type": "priority"}}, "flex", id="native"),
|
||||
pytest.param({"service_tier": "priority"}, "flex", id="openai"),
|
||||
pytest.param({"service_tier": "flex"}, "priority", id="openai-served-priority"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_nova_invoke_stream_reports_and_bills_the_tier_bedrock_served_like_a_non_streaming_call(
|
||||
requested_tier: Mapping[str, object], served_tier: str
|
||||
) -> None:
|
||||
non_streaming, sync_chunks, async_chunks = await _nova_invoke_calls(requested_tier, served_tier)
|
||||
sync_streamed: Final = litellm.stream_chunk_builder(list(sync_chunks))
|
||||
async_streamed: Final = litellm.stream_chunk_builder(list(async_chunks))
|
||||
|
||||
assert {getattr(chunk, "service_tier", None) for chunk in (*sync_chunks, *async_chunks)} == {served_tier}
|
||||
assert sync_streamed.service_tier == async_streamed.service_tier == non_streaming.service_tier == served_tier
|
||||
assert (
|
||||
litellm.completion_cost(sync_streamed)
|
||||
== litellm.completion_cost(async_streamed)
|
||||
== non_streaming._hidden_params["response_cost"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"requested_tier",
|
||||
[
|
||||
pytest.param({"serviceTier": {"type": "priority"}}, id="native"),
|
||||
pytest.param({"service_tier": "priority"}, id="openai"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("served_tier", [pytest.param(None, id="no-header"), pytest.param("", id="empty-header")])
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_nova_invoke_stream_without_a_served_tier_reports_no_tier(
|
||||
requested_tier: Mapping[str, object], served_tier: str | None
|
||||
) -> None:
|
||||
non_streaming, sync_chunks, async_chunks = await _nova_invoke_calls(requested_tier, served_tier)
|
||||
|
||||
assert {getattr(chunk, "service_tier", None) for chunk in (*sync_chunks, *async_chunks)} == {None}
|
||||
assert getattr(litellm.stream_chunk_builder(list(sync_chunks)), "service_tier", None) is None
|
||||
assert getattr(litellm.stream_chunk_builder(list(async_chunks)), "service_tier", None) is None
|
||||
assert getattr(non_streaming, "service_tier", None) is None
|
||||
|
|
|
|||
|
|
@ -3,11 +3,14 @@ Tests for Bedrock Converse API serviceTier support.
|
|||
"""
|
||||
|
||||
import json
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
import litellm
|
||||
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.types.llms.bedrock import ServiceTierBlock
|
||||
|
||||
|
||||
|
|
@ -201,24 +204,165 @@ def test_map_openai_service_tier_flex():
|
|||
assert result["serviceTier"] == {"type": "flex"}
|
||||
|
||||
|
||||
def test_map_openai_service_tier_auto_maps_to_default():
|
||||
"""Test that OpenAI service_tier='auto' maps to Bedrock serviceTier='default'.
|
||||
def test_map_openai_service_tier_auto_sends_no_tier():
|
||||
"""`auto` picks no tier, so Bedrock gets no serviceTier and serves its own default."""
|
||||
config: Final = AmazonConverseConfig()
|
||||
|
||||
Bedrock doesn't support 'auto', so we map it to 'default'.
|
||||
"""
|
||||
config = AmazonConverseConfig()
|
||||
|
||||
result = config.map_openai_params(
|
||||
result: Final = config.map_openai_params(
|
||||
non_default_params={"service_tier": "auto"},
|
||||
optional_params={},
|
||||
model="anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert "serviceTier" in result
|
||||
assert "serviceTier" not in result
|
||||
|
||||
|
||||
def test_map_openai_service_tier_none_sends_no_tier():
|
||||
config: Final = AmazonConverseConfig()
|
||||
|
||||
result: Final = config.map_openai_params(
|
||||
non_default_params={"service_tier": None},
|
||||
optional_params={},
|
||||
model="anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert "serviceTier" not in result
|
||||
|
||||
|
||||
def test_map_openai_service_tier_reserved_reaches_the_request():
|
||||
config: Final = AmazonConverseConfig()
|
||||
model: Final = "anthropic.claude-3-sonnet-20240229-v1:0"
|
||||
|
||||
optional_params: Final = config.map_openai_params(
|
||||
non_default_params={"service_tier": "reserved"},
|
||||
optional_params={},
|
||||
model=model,
|
||||
drop_params=False,
|
||||
)
|
||||
request: Final = config.transform_request(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "Hello!"}],
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["serviceTier"] == {"type": "reserved"}
|
||||
|
||||
|
||||
def test_map_openai_service_tier_standard_only_maps_to_default():
|
||||
"""`standard_only` is the Anthropic Messages value that means Bedrock's Standard tier."""
|
||||
config: Final = AmazonConverseConfig()
|
||||
|
||||
result: Final = config.map_openai_params(
|
||||
non_default_params={"service_tier": "standard_only"},
|
||||
optional_params={},
|
||||
model="anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert result["serviceTier"] == {"type": "default"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "bedrock_tier"),
|
||||
[("standard", "default"), ("Standard_Only", "default"), ("Priority", "priority"), ("FLEX", "flex")],
|
||||
)
|
||||
def test_map_openai_service_tier_maps_standard_and_any_case_to_the_bedrock_tier(value: str, bedrock_tier: str):
|
||||
config: Final = AmazonConverseConfig()
|
||||
|
||||
result: Final = config.map_openai_params(
|
||||
non_default_params={"service_tier": value},
|
||||
optional_params={},
|
||||
model="anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert result["serviceTier"] == {"type": bedrock_tier}
|
||||
|
||||
|
||||
def test_map_openai_service_tier_auto_in_any_case_sends_no_tier():
|
||||
config: Final = AmazonConverseConfig()
|
||||
|
||||
result: Final = config.map_openai_params(
|
||||
non_default_params={"service_tier": "AUTO"},
|
||||
optional_params={},
|
||||
model="anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert "serviceTier" not in result
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["ultrafast", "fast", "scale", "", {"type": "priority"}, ["fast"], 1])
|
||||
def test_map_openai_service_tier_unmapped_value_answers_400(value: object):
|
||||
config: Final = AmazonConverseConfig()
|
||||
optional_params: Final = dict[str, object]()
|
||||
|
||||
with pytest.raises(litellm.utils.UnsupportedParamsError) as error:
|
||||
config.map_openai_params(
|
||||
non_default_params={"service_tier": value},
|
||||
optional_params=optional_params,
|
||||
model="anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert error.value.status_code == 400
|
||||
assert f"service_tier={value!r}" in error.value.message
|
||||
assert "default, flex, priority, reserved" in error.value.message
|
||||
assert "drop_params" in error.value.message
|
||||
assert "serviceTier" not in optional_params
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["ultrafast", {"type": "priority"}])
|
||||
def test_map_openai_service_tier_unmapped_value_drops_under_request_drop_params(value: object):
|
||||
config: Final = AmazonConverseConfig()
|
||||
|
||||
result: Final = config.map_openai_params(
|
||||
non_default_params={"service_tier": value},
|
||||
optional_params={},
|
||||
model="anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert "serviceTier" not in result
|
||||
|
||||
|
||||
def test_map_openai_service_tier_unmapped_value_drops_under_global_drop_params(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
config: Final = AmazonConverseConfig()
|
||||
|
||||
result: Final = config.map_openai_params(
|
||||
non_default_params={"service_tier": "ultrafast"},
|
||||
optional_params={},
|
||||
model="anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert "serviceTier" not in result
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["bedrock/converse/us.openai.gpt-6-luna", "bedrock/converse/us.amazon.nova-2-lite-v1:0"])
|
||||
def test_completion_refuses_unmapped_service_tier_before_calling_bedrock(model: str, monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
client: Final = MagicMock(spec=HTTPHandler)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError) as error:
|
||||
litellm.completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "Hello!"}],
|
||||
service_tier="ultrafast",
|
||||
aws_region_name="us-east-1",
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert error.value.status_code == 400
|
||||
assert "service_tier='ultrafast'" in error.value.message
|
||||
client.post.assert_not_called()
|
||||
|
||||
|
||||
# Tests for service_tier in response
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from contextlib import AsyncExitStack, closing
|
|||
from datetime import datetime, timezone
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
|
@ -98,6 +99,234 @@ class TestBedrockFilesTransformation:
|
|||
assert "_litellm_undeclared_sentinel" not in model_input, model_input
|
||||
assert result[0]["modelInput"]["max_tokens"] == 10
|
||||
|
||||
@staticmethod
|
||||
def _nova_row(
|
||||
url: str = "/v1/chat/completions",
|
||||
model: str = "us.amazon.nova-pro-v1:0",
|
||||
body_params: Mapping[str, object] = MappingProxyType({}),
|
||||
) -> Mapping[str, object]:
|
||||
prompt: Final = {"input": "hi"} if url == "/v1/responses" else {"messages": [{"role": "user", "content": "hi"}]}
|
||||
return {
|
||||
"custom_id": "nova-tier-1",
|
||||
"method": "POST",
|
||||
"url": url,
|
||||
"body": {"model": model, **prompt, **body_params},
|
||||
}
|
||||
|
||||
@pytest.mark.parametrize("tier", [None, "auto", "AUTO"])
|
||||
def test_nova_row_with_null_or_auto_service_tier_sends_no_tier(self, tier: str | None):
|
||||
result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
||||
[self._nova_row(body_params={"service_tier": tier})]
|
||||
)
|
||||
|
||||
assert "serviceTier" not in result[0]["modelInput"]
|
||||
|
||||
@pytest.mark.parametrize("tier", ["ultrafast", "reserved", "priority", "standard_only"])
|
||||
@pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"])
|
||||
def test_nova_row_with_a_service_tier_fails_the_row_without_drop_params(self, url: str, tier: str):
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=f"service_tier='{tier}'"):
|
||||
BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
||||
[self._nova_row(url, body_params={"service_tier": tier})]
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("tier", ["ultrafast", "reserved"])
|
||||
@pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"])
|
||||
def test_nova_row_level_drop_params_drops_the_service_tier(self, url: str, tier: str):
|
||||
result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
||||
[self._nova_row(url, body_params={"service_tier": tier, "drop_params": True})]
|
||||
)
|
||||
|
||||
assert "serviceTier" not in result[0]["modelInput"]
|
||||
assert "drop_params" not in json.dumps(result[0]["modelInput"])
|
||||
|
||||
def test_nova_row_service_tier_drops_under_global_drop_params(self, monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
|
||||
result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
||||
[self._nova_row(body_params={"service_tier": "reserved"})]
|
||||
)
|
||||
|
||||
assert "serviceTier" not in result[0]["modelInput"]
|
||||
|
||||
@pytest.mark.parametrize("tier", ["ultrafast", "reserved"])
|
||||
def test_deployment_drop_params_drops_the_service_tier_from_the_uploaded_nova_rows(self, tier: str):
|
||||
request: Final = BedrockFilesConfig().transform_create_file_request(
|
||||
model="us.amazon.nova-pro-v1:0",
|
||||
create_file_data={
|
||||
"file": (
|
||||
"batch.jsonl",
|
||||
json.dumps(self._nova_row(body_params={"service_tier": tier})).encode(),
|
||||
"application/jsonl",
|
||||
),
|
||||
"purpose": "batch",
|
||||
},
|
||||
optional_params={
|
||||
"aws_access_key_id": "test-key-id",
|
||||
"aws_secret_access_key": "test-secret",
|
||||
"aws_region_name": "us-west-2",
|
||||
},
|
||||
litellm_params={"s3_bucket_name": "litellm-batch-bucket", "drop_params": True},
|
||||
)
|
||||
|
||||
assert isinstance(request, dict)
|
||||
uploaded_row: Final = json.loads(request["data"])
|
||||
assert uploaded_row["modelInput"]["messages"]
|
||||
assert "serviceTier" not in uploaded_row["modelInput"]
|
||||
|
||||
@pytest.mark.parametrize("drop_params, drops", [(False, False), ("false", False), (True, True), ("true", True)])
|
||||
def test_deployment_drop_params_decides_whether_a_claude_row_tier_fails_or_drops(
|
||||
self, drop_params: bool | str, drops: bool
|
||||
):
|
||||
claude_model: Final = "us.anthropic.claude-haiku-4-5-20251001-v1:0"
|
||||
row: Final = {
|
||||
"custom_id": "claude-tier-1",
|
||||
"method": "POST",
|
||||
"url": "/v1/chat/completions",
|
||||
"body": {
|
||||
"model": claude_model,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"max_tokens": 10,
|
||||
"service_tier": "ultrafast",
|
||||
},
|
||||
}
|
||||
|
||||
def upload() -> Mapping[str, object]:
|
||||
request: Final = BedrockFilesConfig().transform_create_file_request(
|
||||
model=claude_model,
|
||||
create_file_data={
|
||||
"file": ("batch.jsonl", json.dumps(row).encode(), "application/jsonl"),
|
||||
"purpose": "batch",
|
||||
},
|
||||
optional_params={
|
||||
"aws_access_key_id": "test-key-id",
|
||||
"aws_secret_access_key": "test-secret",
|
||||
"aws_region_name": "us-west-2",
|
||||
},
|
||||
litellm_params={"s3_bucket_name": "litellm-batch-bucket", "drop_params": drop_params},
|
||||
)
|
||||
assert isinstance(request, dict)
|
||||
return json.loads(request["data"])
|
||||
|
||||
if not drops:
|
||||
with pytest.raises(BedrockError, match="service_tier"):
|
||||
upload()
|
||||
return
|
||||
|
||||
model_input: Final = upload()["modelInput"]
|
||||
assert model_input["messages"]
|
||||
assert "service_tier" not in json.dumps(model_input)
|
||||
assert "serviceTier" not in model_input
|
||||
|
||||
@pytest.mark.parametrize("tier", [None, "auto", "AUTO"])
|
||||
def test_claude_row_with_null_or_auto_service_tier_sends_no_tier(self, tier: str | None):
|
||||
result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
||||
[
|
||||
self._nova_row(
|
||||
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
body_params={"service_tier": tier, "max_tokens": 10},
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
assert result[0]["modelInput"]["messages"]
|
||||
assert "service_tier" not in json.dumps(result[0]["modelInput"])
|
||||
|
||||
@pytest.mark.parametrize("drop_params, drops", [(True, True), ("true", True), (False, False), ("false", False)])
|
||||
@pytest.mark.parametrize(
|
||||
"model", ["us.anthropic.claude-haiku-4-5-20251001-v1:0", "openai.gpt-oss-120b-1:0", "us.amazon.nova-pro-v1:0"]
|
||||
)
|
||||
def test_row_level_drop_params_decides_whether_a_rejected_tier_fails_or_drops_for_every_row_format(
|
||||
self, model: str, drop_params: bool | str, drops: bool
|
||||
):
|
||||
rows: Final = [
|
||||
self._nova_row(
|
||||
model=model, body_params={"service_tier": "ultrafast", "drop_params": drop_params, "max_tokens": 10}
|
||||
)
|
||||
]
|
||||
|
||||
if not drops:
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match="service_tier='ultrafast'"):
|
||||
BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(rows)
|
||||
return
|
||||
|
||||
result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(rows)
|
||||
assert result[0]["modelInput"]["messages"]
|
||||
assert "ultrafast" not in json.dumps(result[0]["modelInput"])
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tier, bedrock_tier",
|
||||
[("priority", "priority"), ("Flex", "flex"), ("standard", "default"), ("default", "default")],
|
||||
)
|
||||
@pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"])
|
||||
def test_openai_format_row_forwards_a_tier_its_invoke_body_takes(self, url: str, tier: str, bedrock_tier: str):
|
||||
result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
||||
[self._nova_row(url, model="openai.gpt-oss-120b-1:0", body_params={"service_tier": tier})]
|
||||
)
|
||||
|
||||
assert result[0]["modelInput"]["service_tier"] == bedrock_tier
|
||||
|
||||
@pytest.mark.parametrize("tier", [None, "auto", "AUTO"])
|
||||
def test_openai_format_row_with_null_or_auto_service_tier_sends_no_tier(self, tier: str | None):
|
||||
result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
||||
[self._nova_row(model="openai.gpt-oss-120b-1:0", body_params={"service_tier": tier})]
|
||||
)
|
||||
|
||||
assert "service_tier" not in result[0]["modelInput"]
|
||||
|
||||
@pytest.mark.parametrize("tier", ["ultrafast", "reserved", "scale"])
|
||||
@pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"])
|
||||
def test_openai_format_row_with_a_tier_bedrock_rejects_fails_without_drop_params(self, url: str, tier: str):
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=f"service_tier='{tier}'"):
|
||||
BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
||||
[self._nova_row(url, model="openai.gpt-oss-120b-1:0", body_params={"service_tier": tier})]
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("url", ["/v1/chat/completions", "/v1/responses"])
|
||||
def test_passthrough_row_keeps_row_level_drop_params_out_of_the_model_input(self, url: str):
|
||||
result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
||||
[self._nova_row(url, "openai.gpt-oss-120b-1:0", {"drop_params": True})]
|
||||
)
|
||||
|
||||
assert "drop_params" not in result[0]["modelInput"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"row_drop_params, deployment_drop_params, drops",
|
||||
[(True, False, True), ("true", False, True), (None, True, True), (None, False, False), ("false", False, False)],
|
||||
)
|
||||
def test_drop_params_decides_whether_a_param_nova_rejects_fails_or_drops_from_an_uploaded_nova_row(
|
||||
self, row_drop_params: bool | str | None, deployment_drop_params: bool, drops: bool
|
||||
):
|
||||
tool: Final = {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}
|
||||
row_params: Final = {
|
||||
"tools": [tool],
|
||||
"tool_choice": "none",
|
||||
**({} if row_drop_params is None else {"drop_params": row_drop_params}),
|
||||
}
|
||||
row_bytes: Final = json.dumps(self._nova_row(body_params=row_params)).encode()
|
||||
|
||||
def upload() -> Mapping[str, object]:
|
||||
request: Final = BedrockFilesConfig().transform_create_file_request(
|
||||
model="us.amazon.nova-pro-v1:0",
|
||||
create_file_data={"file": ("batch.jsonl", row_bytes, "application/jsonl"), "purpose": "batch"},
|
||||
optional_params={
|
||||
"aws_access_key_id": "test-key-id",
|
||||
"aws_secret_access_key": "test-secret",
|
||||
"aws_region_name": "us-west-2",
|
||||
},
|
||||
litellm_params={"s3_bucket_name": "litellm-batch-bucket", "drop_params": deployment_drop_params},
|
||||
)
|
||||
assert isinstance(request, dict)
|
||||
return json.loads(request["data"])
|
||||
|
||||
if not drops:
|
||||
with pytest.raises(BedrockError, match="tool_choice"):
|
||||
upload()
|
||||
return
|
||||
|
||||
tool_config: Final = upload()["modelInput"]["toolConfig"]
|
||||
assert tool_config["tools"]
|
||||
assert "toolChoice" not in tool_config
|
||||
|
||||
def test_nova_text_only_uses_converse_format(self):
|
||||
"""
|
||||
Test that Nova models produce Converse API format in batch modelInput.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue