mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-13 08:52:02 +00:00
fix(bedrock): answer 400 for service tiers Bedrock cannot serve, forward reserved
Converse used to drop any service_tier outside default, flex, and priority with no error, rewrite auto to default, and drop reserved. Now reserved and standard_only map to Bedrock tiers, auto sends no tier, and any other value answers 400 unless drop_params is on. Nova batch rows honor the row's drop_params.
This commit is contained in:
parent
d947ec1f7d
commit
ce94d2894d
5 changed files with 178 additions and 21 deletions
|
|
@ -202,6 +202,8 @@ def _tools_the_model_accepts(
|
||||||
|
|
||||||
_OUTPUT_CONFIG_BLOCK: Final = TypeAdapter(OutputConfigBlock)
|
_OUTPUT_CONFIG_BLOCK: Final = TypeAdapter(OutputConfigBlock)
|
||||||
_DEEPSEEK_THINKING_REASON: Final = "DeepSeek on Bedrock reasons natively and takes no thinking field."
|
_DEEPSEEK_THINKING_REASON: Final = "DeepSeek on Bedrock reasons natively and takes no thinking field."
|
||||||
|
BEDROCK_SERVICE_TIERS: Final = ("default", "flex", "priority", "reserved")
|
||||||
|
_BEDROCK_SERVICE_TIER_ALIASES: Final = types.MappingProxyType({"standard_only": "default"})
|
||||||
_DEEPSEEK_R1_EFFORT_REASON: Final = "DeepSeek R1 on Bedrock always reasons and rejects reasoning_effort."
|
_DEEPSEEK_R1_EFFORT_REASON: Final = "DeepSeek R1 on Bedrock always reasons and rejects reasoning_effort."
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1250,8 +1252,13 @@ class AmazonConverseConfig(BaseConfig):
|
||||||
self._map_context_management_param(value, optional_params)
|
self._map_context_management_param(value, optional_params)
|
||||||
if param == "requestMetadata":
|
if param == "requestMetadata":
|
||||||
self._map_request_metadata_param(value, optional_params, drop_params)
|
self._map_request_metadata_param(value, optional_params, drop_params)
|
||||||
if param == "service_tier" and isinstance(value, str):
|
if param == "service_tier":
|
||||||
self._map_service_tier_param(value, optional_params)
|
self._map_service_tier_param(
|
||||||
|
REQUEST_PARAMS.validate_python(request_params)["service_tier"],
|
||||||
|
optional_params,
|
||||||
|
model,
|
||||||
|
drop_params,
|
||||||
|
)
|
||||||
|
|
||||||
if param == "web_search_options" and isinstance(value, dict):
|
if param == "web_search_options" and isinstance(value, dict):
|
||||||
# Note: we use `isinstance(value, dict)` instead of `value and isinstance(value, dict)`
|
# Note: we use `isinstance(value, dict)` instead of `value and isinstance(value, dict)`
|
||||||
|
|
@ -1305,17 +1312,22 @@ class AmazonConverseConfig(BaseConfig):
|
||||||
if mapped is not None:
|
if mapped is not None:
|
||||||
optional_params["context_management"] = mapped
|
optional_params["context_management"] = mapped
|
||||||
|
|
||||||
def _map_service_tier_param(self, value: str, optional_params: dict) -> None:
|
def _map_service_tier_param(self, value: object, optional_params: dict, model: str, drop_params: bool) -> None:
|
||||||
"""Map OpenAI service_tier (string) to Bedrock serviceTier (object).
|
if value is None or value == "auto":
|
||||||
|
return
|
||||||
OpenAI values: "auto", "default", "flex", "priority"
|
bedrock_tier: Final = _BEDROCK_SERVICE_TIER_ALIASES.get(value, value) if isinstance(value, str) else None
|
||||||
Bedrock values: "default", "flex", "priority" (no "auto")
|
if bedrock_tier in BEDROCK_SERVICE_TIERS:
|
||||||
"""
|
|
||||||
bedrock_tier = value
|
|
||||||
if value == "auto":
|
|
||||||
bedrock_tier = "default" # Bedrock doesn't support "auto"
|
|
||||||
if bedrock_tier in ("default", "flex", "priority"):
|
|
||||||
optional_params["serviceTier"] = {"type": bedrock_tier}
|
optional_params["serviceTier"] = {"type": bedrock_tier}
|
||||||
|
return
|
||||||
|
if not (litellm.drop_params or drop_params):
|
||||||
|
raise litellm.utils.UnsupportedParamsError(
|
||||||
|
message=(
|
||||||
|
f"{model} does not support service_tier={value!r} on Bedrock. "
|
||||||
|
"To drop unsupported params, set `litellm.drop_params = True`."
|
||||||
|
),
|
||||||
|
status_code=400,
|
||||||
|
)
|
||||||
|
verbose_logger.warning("Dropping service_tier=%r for %s, since Bedrock does not serve it.", value, model)
|
||||||
|
|
||||||
def _translate_response_format_param(
|
def _translate_response_format_param(
|
||||||
self,
|
self,
|
||||||
|
|
|
||||||
|
|
@ -1105,7 +1105,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
||||||
non_default_params={k: v for k, v in optional_params.items() if k != "requestMetadata"},
|
non_default_params={k: v for k, v in optional_params.items() if k != "requestMetadata"},
|
||||||
optional_params={},
|
optional_params={},
|
||||||
model=model,
|
model=model,
|
||||||
drop_params=False,
|
drop_params=optional_params.get("drop_params") is True,
|
||||||
)
|
)
|
||||||
return converse_config.transform_request(
|
return converse_config.transform_request(
|
||||||
model=model,
|
model=model,
|
||||||
|
|
|
||||||
|
|
@ -244,7 +244,7 @@ class ConverseTokenUsageBlock(TypedDict, total=False):
|
||||||
|
|
||||||
|
|
||||||
class ServiceTierBlock(TypedDict):
|
class ServiceTierBlock(TypedDict):
|
||||||
type: Literal["priority", "default", "flex"]
|
type: Literal["priority", "default", "flex", "reserved"]
|
||||||
|
|
||||||
|
|
||||||
class ConverseResponseBlock(TypedDict, total=False):
|
class ConverseResponseBlock(TypedDict, total=False):
|
||||||
|
|
|
||||||
|
|
@ -3,11 +3,13 @@ Tests for Bedrock Converse API serviceTier support.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
import litellm
|
||||||
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
|
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
|
||||||
|
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||||
from litellm.types.llms.bedrock import ServiceTierBlock
|
from litellm.types.llms.bedrock import ServiceTierBlock
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -201,11 +203,8 @@ def test_map_openai_service_tier_flex():
|
||||||
assert result["serviceTier"] == {"type": "flex"}
|
assert result["serviceTier"] == {"type": "flex"}
|
||||||
|
|
||||||
|
|
||||||
def test_map_openai_service_tier_auto_maps_to_default():
|
def test_map_openai_service_tier_auto_sends_no_tier():
|
||||||
"""Test that OpenAI service_tier='auto' maps to Bedrock serviceTier='default'.
|
"""`auto` picks no tier, so Bedrock gets no serviceTier and serves its own default."""
|
||||||
|
|
||||||
Bedrock doesn't support 'auto', so we map it to 'default'.
|
|
||||||
"""
|
|
||||||
config = AmazonConverseConfig()
|
config = AmazonConverseConfig()
|
||||||
|
|
||||||
result = config.map_openai_params(
|
result = config.map_openai_params(
|
||||||
|
|
@ -215,10 +214,123 @@ def test_map_openai_service_tier_auto_maps_to_default():
|
||||||
drop_params=False,
|
drop_params=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert "serviceTier" in result
|
assert "serviceTier" not in result
|
||||||
|
|
||||||
|
|
||||||
|
def test_map_openai_service_tier_none_sends_no_tier():
|
||||||
|
config = AmazonConverseConfig()
|
||||||
|
|
||||||
|
result = config.map_openai_params(
|
||||||
|
non_default_params={"service_tier": None},
|
||||||
|
optional_params={},
|
||||||
|
model="anthropic.claude-3-sonnet-20240229-v1:0",
|
||||||
|
drop_params=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "serviceTier" not in result
|
||||||
|
|
||||||
|
|
||||||
|
def test_map_openai_service_tier_reserved_reaches_the_request():
|
||||||
|
config = AmazonConverseConfig()
|
||||||
|
model = "anthropic.claude-3-sonnet-20240229-v1:0"
|
||||||
|
|
||||||
|
optional_params = config.map_openai_params(
|
||||||
|
non_default_params={"service_tier": "reserved"},
|
||||||
|
optional_params={},
|
||||||
|
model=model,
|
||||||
|
drop_params=False,
|
||||||
|
)
|
||||||
|
request = config.transform_request(
|
||||||
|
model=model,
|
||||||
|
messages=[{"role": "user", "content": "Hello!"}],
|
||||||
|
optional_params=optional_params,
|
||||||
|
litellm_params={},
|
||||||
|
headers={},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert request["serviceTier"] == {"type": "reserved"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_map_openai_service_tier_standard_only_maps_to_default():
|
||||||
|
"""`standard_only` is the Anthropic Messages value that means Bedrock's Standard tier."""
|
||||||
|
config = AmazonConverseConfig()
|
||||||
|
|
||||||
|
result = config.map_openai_params(
|
||||||
|
non_default_params={"service_tier": "standard_only"},
|
||||||
|
optional_params={},
|
||||||
|
model="anthropic.claude-3-sonnet-20240229-v1:0",
|
||||||
|
drop_params=False,
|
||||||
|
)
|
||||||
|
|
||||||
assert result["serviceTier"] == {"type": "default"}
|
assert result["serviceTier"] == {"type": "default"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("value", ["ultrafast", "fast", "scale", "", {"type": "priority"}, ["fast"], 1])
|
||||||
|
def test_map_openai_service_tier_unmapped_value_answers_400(value):
|
||||||
|
config = AmazonConverseConfig()
|
||||||
|
optional_params = {}
|
||||||
|
|
||||||
|
with pytest.raises(litellm.utils.UnsupportedParamsError) as error:
|
||||||
|
config.map_openai_params(
|
||||||
|
non_default_params={"service_tier": value},
|
||||||
|
optional_params=optional_params,
|
||||||
|
model="anthropic.claude-3-sonnet-20240229-v1:0",
|
||||||
|
drop_params=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert error.value.status_code == 400
|
||||||
|
assert f"service_tier={value!r}" in error.value.message
|
||||||
|
assert "drop_params" in error.value.message
|
||||||
|
assert "serviceTier" not in optional_params
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("value", ["ultrafast", {"type": "priority"}])
|
||||||
|
def test_map_openai_service_tier_unmapped_value_drops_under_request_drop_params(value):
|
||||||
|
config = AmazonConverseConfig()
|
||||||
|
|
||||||
|
result = config.map_openai_params(
|
||||||
|
non_default_params={"service_tier": value},
|
||||||
|
optional_params={},
|
||||||
|
model="anthropic.claude-3-sonnet-20240229-v1:0",
|
||||||
|
drop_params=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "serviceTier" not in result
|
||||||
|
|
||||||
|
|
||||||
|
def test_map_openai_service_tier_unmapped_value_drops_under_global_drop_params(monkeypatch):
|
||||||
|
monkeypatch.setattr(litellm, "drop_params", True)
|
||||||
|
config = AmazonConverseConfig()
|
||||||
|
|
||||||
|
result = config.map_openai_params(
|
||||||
|
non_default_params={"service_tier": "ultrafast"},
|
||||||
|
optional_params={},
|
||||||
|
model="anthropic.claude-3-sonnet-20240229-v1:0",
|
||||||
|
drop_params=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "serviceTier" not in result
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("model", ["bedrock/converse/us.openai.gpt-6-luna", "bedrock/converse/us.amazon.nova-2-lite-v1:0"])
|
||||||
|
def test_completion_refuses_unmapped_service_tier_before_calling_bedrock(model, monkeypatch):
|
||||||
|
monkeypatch.setattr(litellm, "drop_params", False)
|
||||||
|
client = MagicMock(spec=HTTPHandler)
|
||||||
|
|
||||||
|
with pytest.raises(litellm.BadRequestError) as error:
|
||||||
|
litellm.completion(
|
||||||
|
model=model,
|
||||||
|
messages=[{"role": "user", "content": "Hello!"}],
|
||||||
|
service_tier="ultrafast",
|
||||||
|
aws_region_name="us-east-1",
|
||||||
|
client=client,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert error.value.status_code == 400
|
||||||
|
assert "service_tier='ultrafast'" in error.value.message
|
||||||
|
client.post.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
# Tests for service_tier in response
|
# Tests for service_tier in response
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -99,6 +99,39 @@ class TestBedrockFilesTransformation:
|
||||||
assert "_litellm_undeclared_sentinel" not in model_input, model_input
|
assert "_litellm_undeclared_sentinel" not in model_input, model_input
|
||||||
assert result[0]["modelInput"]["max_tokens"] == 10
|
assert result[0]["modelInput"]["max_tokens"] == 10
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _nova_row(**body_params: object) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"custom_id": "nova-tier-1",
|
||||||
|
"method": "POST",
|
||||||
|
"url": "/v1/chat/completions",
|
||||||
|
"body": {
|
||||||
|
"model": "us.amazon.nova-pro-v1:0",
|
||||||
|
"messages": [{"role": "user", "content": "hi"}],
|
||||||
|
**body_params,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_nova_row_with_null_service_tier_sends_no_tier(self):
|
||||||
|
result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
||||||
|
[self._nova_row(service_tier=None)]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "serviceTier" not in result[0]["modelInput"]
|
||||||
|
|
||||||
|
def test_nova_row_with_unmapped_service_tier_fails_the_row_like_real_time(self):
|
||||||
|
with pytest.raises(litellm.UnsupportedParamsError, match="ultrafast"):
|
||||||
|
BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
||||||
|
[self._nova_row(service_tier="ultrafast")]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_nova_row_level_drop_params_drops_the_unmapped_service_tier(self):
|
||||||
|
result: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
||||||
|
[self._nova_row(service_tier="ultrafast", drop_params=True)]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "serviceTier" not in result[0]["modelInput"]
|
||||||
|
|
||||||
def test_nova_text_only_uses_converse_format(self):
|
def test_nova_text_only_uses_converse_format(self):
|
||||||
"""
|
"""
|
||||||
Test that Nova models produce Converse API format in batch modelInput.
|
Test that Nova models produce Converse API format in batch modelInput.
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue