mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge aa6483b2b3 into c4b01b2ac8
This commit is contained in:
commit
49526f1964
6 changed files with 276 additions and 101 deletions
|
|
@ -42,6 +42,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
bedrock_model_is_openai_gpt,
|
||||
bedrock_rejects_stop_sequences,
|
||||
bedrock_runtime_chat_completions_serves_reasoning_inline,
|
||||
bedrock_serves_service_tier,
|
||||
split_bedrock_region_path,
|
||||
strip_bedrock_routing_prefix,
|
||||
)
|
||||
|
|
@ -318,6 +319,13 @@ def with_max_completion_tokens(params: Mapping[str, object]) -> Mapping[str, obj
|
|||
)
|
||||
|
||||
|
||||
def _unsupported_service_tier(model: str, raw_params: Mapping[str, object]) -> frozenset[str]:
|
||||
service_tier: Final = raw_params.get("service_tier")
|
||||
if not isinstance(service_tier, str) or bedrock_serves_service_tier(model, service_tier):
|
||||
return frozenset()
|
||||
return frozenset(("service_tier",))
|
||||
|
||||
|
||||
class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
|
||||
def __init__(self, aws_signer: BaseAWSLLM | None = None) -> None:
|
||||
super().__init__()
|
||||
|
|
@ -463,11 +471,22 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
|
|||
drop_params_gated=True,
|
||||
)
|
||||
rejected_stop: Final = frozenset(("stop",)) if bedrock_rejects_stop_sequences(model) else frozenset[str]()
|
||||
unsupported_tier: Final = _unsupported_service_tier(model, raw_params)
|
||||
if unsupported_tier and not (litellm.drop_params or drop_params):
|
||||
raise litellm.utils.UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} does not support service_tier={raw_params['service_tier']!r} on Bedrock. "
|
||||
"To drop unsupported params, set `litellm.drop_params = True`."
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
return dict(
|
||||
without_refused_reasoning_effort(
|
||||
model,
|
||||
with_max_completion_tokens(
|
||||
_without_params(mapped, refused_while_reasoning | malformed_effort | rejected_stop)
|
||||
_without_params(
|
||||
mapped, refused_while_reasoning | malformed_effort | rejected_stop | unsupported_tier
|
||||
)
|
||||
),
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
bedrock_model_supports_regex_lookaround,
|
||||
bedrock_native_output_format,
|
||||
bedrock_native_output_is_opt_in,
|
||||
bedrock_serves_service_tier,
|
||||
bedrock_supports_tool_search,
|
||||
deployment_model_info,
|
||||
requests_output_config_format,
|
||||
|
|
@ -1215,7 +1216,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
if param == "requestMetadata":
|
||||
self._map_request_metadata_param(value, optional_params, drop_params)
|
||||
if param == "service_tier" and isinstance(value, str):
|
||||
self._map_service_tier_param(value, optional_params)
|
||||
self._map_service_tier_param(value, optional_params, model=model, drop_params=drop_params)
|
||||
|
||||
if param == "web_search_options" and isinstance(value, dict):
|
||||
# Note: we use `isinstance(value, dict)` instead of `value and isinstance(value, dict)`
|
||||
|
|
@ -1269,17 +1270,27 @@ class AmazonConverseConfig(BaseConfig):
|
|||
if mapped is not None:
|
||||
optional_params["context_management"] = mapped
|
||||
|
||||
def _map_service_tier_param(self, value: str, optional_params: dict) -> None:
|
||||
def _map_service_tier_param(self, value: str, optional_params: dict, model: str, drop_params: bool) -> None:
|
||||
"""Map OpenAI service_tier (string) to Bedrock serviceTier (object).
|
||||
|
||||
OpenAI values: "auto", "default", "flex", "priority"
|
||||
Bedrock values: "default", "flex", "priority" (no "auto")
|
||||
"""
|
||||
bedrock_tier = value
|
||||
if value == "auto":
|
||||
bedrock_tier = "default" # Bedrock doesn't support "auto"
|
||||
if bedrock_tier in ("default", "flex", "priority"):
|
||||
optional_params["serviceTier"] = {"type": bedrock_tier}
|
||||
bedrock_tier: Final = "default" if value == "auto" else value
|
||||
if bedrock_tier not in ("default", "flex", "priority"):
|
||||
return
|
||||
if not bedrock_serves_service_tier(model, bedrock_tier):
|
||||
if not (litellm.drop_params or drop_params):
|
||||
raise litellm.utils.UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} does not support service_tier={value!r} on Bedrock. "
|
||||
"To drop unsupported params, set `litellm.drop_params = True`."
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
verbose_logger.debug("Dropping unsupported `service_tier=%s` for Bedrock model=%s.", value, model)
|
||||
return
|
||||
optional_params["serviceTier"] = {"type": bedrock_tier}
|
||||
|
||||
def _translate_response_format_param(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -1325,6 +1325,30 @@ def _mantle_api_base_from_env() -> str | None:
|
|||
return next((base[: -len(suffix)] for suffix in _MANTLE_OPENAI_BASE_SUFFIXES if base.endswith(suffix)), base)
|
||||
|
||||
|
||||
def bedrock_serves_service_tier(model: str, service_tier: str) -> bool:
|
||||
"""Whether AWS sells this Bedrock model at the requested ``service_tier``.
|
||||
|
||||
AWS enables priority and flex per model, not per provider (Nova 2, gpt-oss and Kimi serve
|
||||
them, every Claude and GPT 5.6 Terra answer ``The provided service tier is not supported for
|
||||
this model``, checked live on 2026-10-27). The price map carries a ``*_cost_per_token_<tier>``
|
||||
row for exactly the models that serve a tier, so a non-null price is the capability signal (the
|
||||
proxy normalizes every row to the full ModelInfo shape, so key presence alone means nothing). ``default``
|
||||
and ``auto`` are always served, and a model absent from the price map has no signal and is
|
||||
forwarded as before. Regional CRIS rows (``eu.``, ``apac.``, ``us-gov.``) often omit the tier
|
||||
prices their base row carries, so the base row is consulted too.
|
||||
"""
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import get_service_tier_cost_key
|
||||
|
||||
cost_key: Final = get_service_tier_cost_key("input_cost_per_token", service_tier)
|
||||
if cost_key == "input_cost_per_token":
|
||||
return True
|
||||
candidates: Final = (*_bedrock_price_map_entries(model), _model_cost_entry(get_bedrock_base_model(model)))
|
||||
entries: Final = tuple(entry for entry in candidates if entry is not None)
|
||||
if not entries:
|
||||
return True
|
||||
return any(entry.get(cost_key) is not None for entry in entries)
|
||||
|
||||
|
||||
DROP_DISABLED_REASONING_EFFORT_WARNING: Final = (
|
||||
"Dropping %s=%r for Bedrock model=%r (drop_params=True) because the model map marks that level unsupported. "
|
||||
"Send a level the model supports."
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
|
@ -657,7 +658,7 @@ def _assert_remote_images_inlined(content):
|
|||
|
||||
|
||||
def test_transform_request_inlines_remote_image_urls(local_cost_map, monkeypatch):
|
||||
import litellm.litellm_core_utils.prompt_templates.image_handling as image_handling
|
||||
from litellm.litellm_core_utils.prompt_templates import image_handling
|
||||
|
||||
monkeypatch.setattr(image_handling, "convert_url_to_base64", lambda url: f"data:image/png;base64,{url}")
|
||||
body = AmazonBedrockRuntimeChatCompletionsConfig().transform_request(
|
||||
|
|
@ -672,7 +673,7 @@ def test_transform_request_inlines_remote_image_urls(local_cost_map, monkeypatch
|
|||
|
||||
|
||||
async def test_async_transform_request_inlines_remote_image_urls(local_cost_map, monkeypatch):
|
||||
import litellm.litellm_core_utils.prompt_templates.image_handling as image_handling
|
||||
from litellm.litellm_core_utils.prompt_templates import image_handling
|
||||
|
||||
async def fake_convert(url, timeout=None):
|
||||
return f"data:image/png;base64,{url}"
|
||||
|
|
@ -1724,3 +1725,53 @@ def test_positional_tool_call_ids_are_minted_for_gpt_too(local_cost_map, fake_aw
|
|||
ids = [tool_call.id for tool_call in response.choices[0].message.tool_calls]
|
||||
assert len(set(ids)) == 2
|
||||
assert all(re.fullmatch(r"call_[0-9a-f]{32}", tool_call_id) for tool_call_id in ids)
|
||||
|
||||
|
||||
# GPT 5.6 Terra on Bedrock answers priority and flex with `'priority' is not supported for 'service_tier'
|
||||
# on this model`, checked live on 2026-10-27 (LIT-9793). Its price-map row carries null tier prices to match.
|
||||
def test_native_chat_completions_drops_service_tier_the_model_does_not_sell(
|
||||
local_cost_map: object, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
config: Final = AmazonBedrockRuntimeChatCompletionsConfig()
|
||||
|
||||
result: Final = config.map_openai_params(
|
||||
non_default_params={"service_tier": "priority", "max_tokens": 5},
|
||||
optional_params={},
|
||||
model="us.openai.gpt-5.6-terra",
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert "service_tier" not in result
|
||||
assert result["max_completion_tokens"] == 5
|
||||
|
||||
|
||||
def test_native_chat_completions_raises_on_service_tier_the_model_does_not_sell(
|
||||
local_cost_map: object, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
config: Final = AmazonBedrockRuntimeChatCompletionsConfig()
|
||||
|
||||
with pytest.raises(litellm.utils.UnsupportedParamsError, match="service_tier='priority'"):
|
||||
config.map_openai_params(
|
||||
non_default_params={"service_tier": "priority"},
|
||||
optional_params={},
|
||||
model="us.openai.gpt-5.6-terra",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
|
||||
def test_native_chat_completions_keeps_default_service_tier(
|
||||
local_cost_map: object, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
config: Final = AmazonBedrockRuntimeChatCompletionsConfig()
|
||||
|
||||
result: Final = config.map_openai_params(
|
||||
non_default_params={"service_tier": "default"},
|
||||
optional_params={},
|
||||
model="us.openai.gpt-5.6-terra",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert result["service_tier"] == "default"
|
||||
|
|
|
|||
|
|
@ -11097,6 +11097,71 @@ def test_a_deployment_with_a_batch_control_plane_endpoint_keeps_it_out_of_chat_r
|
|||
assert "aws_bedrock_control_plane_endpoint" not in route.calls[0].request.content.decode()
|
||||
|
||||
|
||||
# Bedrock tier support is per model. Checked live on 2026-10-27 (LIT-9793): Claude Sonnet 4.5 and
|
||||
# Haiku 4.5 answer priority and flex with `The provided service tier is not supported for this model.`,
|
||||
# Nova 2 Lite serves both. The price map mirrors that, Claude rows have null tier prices, Nova 2 has real ones.
|
||||
def test_converse_service_tier_dropped_on_models_without_that_tier_under_drop_params() -> None:
|
||||
config: Final = AmazonConverseConfig()
|
||||
|
||||
result: Final = config.map_openai_params(
|
||||
non_default_params={"service_tier": "priority"},
|
||||
optional_params={},
|
||||
model="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert "serviceTier" not in result
|
||||
|
||||
|
||||
def test_converse_service_tier_raises_on_models_without_that_tier_without_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
config: Final = AmazonConverseConfig()
|
||||
|
||||
with pytest.raises(litellm.utils.UnsupportedParamsError, match="service_tier='flex'"):
|
||||
config.map_openai_params(
|
||||
non_default_params={"service_tier": "flex"},
|
||||
optional_params={},
|
||||
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("service_tier", ["priority", "flex"])
|
||||
def test_converse_service_tier_forwarded_on_models_that_sell_it(
|
||||
monkeypatch: pytest.MonkeyPatch, service_tier: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
config: Final = AmazonConverseConfig()
|
||||
|
||||
result: Final = config.map_openai_params(
|
||||
non_default_params={"service_tier": service_tier},
|
||||
optional_params={},
|
||||
model="us.amazon.nova-2-lite-v1:0",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert result["serviceTier"] == {"type": service_tier}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("service_tier", ["default", "auto"])
|
||||
def test_converse_default_and_auto_service_tier_always_forwarded(
|
||||
monkeypatch: pytest.MonkeyPatch, service_tier: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
config: Final = AmazonConverseConfig()
|
||||
|
||||
result: Final = config.map_openai_params(
|
||||
non_default_params={"service_tier": service_tier},
|
||||
optional_params={},
|
||||
model="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert result["serviceTier"] == {"type": "default"}
|
||||
|
||||
|
||||
def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage()
|
||||
|
|
|
|||
|
|
@ -1,13 +1,19 @@
|
|||
|
||||
import asyncio, importlib, litellm, pytest
|
||||
import asyncio
|
||||
import importlib
|
||||
from collections.abc import Iterator
|
||||
from typing import Final
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.bedrock.common_utils import(
|
||||
import litellm
|
||||
from litellm import completion
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
AmazonBedrockGlobalConfig,
|
||||
BedrockModelInfo,
|
||||
NativeOutputFallback,
|
||||
bedrock_native_output_is_opt_in,
|
||||
bedrock_serves_service_tier,
|
||||
bedrock_strict_tools_fallback,
|
||||
deployment_model_info,
|
||||
extract_model_name_from_bedrock_arn,
|
||||
|
|
@ -18,11 +24,8 @@ from litellm.llms.bedrock.common_utils import(
|
|||
strip_bedrock_throughput_suffix,
|
||||
without_bedrock_native_output_opt_in,
|
||||
)
|
||||
from collections.abc import Iterator
|
||||
from litellm import completion
|
||||
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
|
||||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# get_bedrock_response_stream_shape lazy-load tests #
|
||||
|
|
@ -50,9 +53,7 @@ def test_bedrock_response_stream_shape_lazy_loads_once():
|
|||
import litellm.llms.bedrock.common_utils as mod
|
||||
|
||||
sentinel = MagicMock()
|
||||
with patch.object(
|
||||
mod, "_load_bedrock_response_stream_shape", return_value=sentinel
|
||||
) as mock_load:
|
||||
with patch.object(mod, "_load_bedrock_response_stream_shape", return_value=sentinel) as mock_load:
|
||||
assert mod.get_bedrock_response_stream_shape() is sentinel
|
||||
assert mod.get_bedrock_response_stream_shape() is sentinel
|
||||
mock_load.assert_called_once()
|
||||
|
|
@ -99,9 +100,7 @@ def test_bedrock_response_stream_shape_is_structure_shape():
|
|||
from litellm.llms.bedrock.common_utils import get_bedrock_response_stream_shape
|
||||
|
||||
loaded_shape = get_bedrock_response_stream_shape()
|
||||
assert (
|
||||
loaded_shape is not None
|
||||
), "get_bedrock_response_stream_shape() is None — botocore may not be installed"
|
||||
assert loaded_shape is not None, "get_bedrock_response_stream_shape() is None — botocore may not be installed"
|
||||
shape: StructureShape = loaded_shape
|
||||
assert isinstance(shape, StructureShape)
|
||||
assert shape.name == "ResponseStream"
|
||||
|
|
@ -166,9 +165,7 @@ def test_deepseek_cris():
|
|||
Test that DeepSeek models with cross-region inference prefix use converse route
|
||||
"""
|
||||
bedrock_model_info = BedrockModelInfo
|
||||
bedrock_route = bedrock_model_info.get_bedrock_route(
|
||||
model="bedrock/us.deepseek.r1-v1:0"
|
||||
)
|
||||
bedrock_route = bedrock_model_info.get_bedrock_route(model="bedrock/us.deepseek.r1-v1:0")
|
||||
assert bedrock_route == "converse"
|
||||
|
||||
|
||||
|
|
@ -234,27 +231,19 @@ def test_govcloud_cross_region_inference_prefix():
|
|||
bedrock_model_info = BedrockModelInfo
|
||||
|
||||
# Test us-gov prefix is stripped correctly for Claude models
|
||||
base_model = bedrock_model_info.get_base_model(
|
||||
model="bedrock/us-gov.anthropic.claude-haiku-4-5-20251001-v1:0"
|
||||
)
|
||||
base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-haiku-4-5-20251001-v1:0")
|
||||
assert base_model == "anthropic.claude-haiku-4-5-20251001-v1:0"
|
||||
|
||||
# Test us-gov prefix is stripped correctly for different Claude versions
|
||||
base_model = bedrock_model_info.get_base_model(
|
||||
model="bedrock/us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
)
|
||||
base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0")
|
||||
assert base_model == "anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
|
||||
# Test us-gov prefix is stripped correctly for Haiku models
|
||||
base_model = bedrock_model_info.get_base_model(
|
||||
model="bedrock/us-gov.anthropic.claude-3-haiku-20240307-v1:0"
|
||||
)
|
||||
base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-3-haiku-20240307-v1:0")
|
||||
assert base_model == "anthropic.claude-3-haiku-20240307-v1:0"
|
||||
|
||||
# Test us-gov prefix is stripped correctly for Meta models
|
||||
base_model = bedrock_model_info.get_base_model(
|
||||
model="bedrock/us-gov.meta.llama3-8b-instruct-v1:0"
|
||||
)
|
||||
base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.meta.llama3-8b-instruct-v1:0")
|
||||
assert base_model == "meta.llama3-8b-instruct-v1:0"
|
||||
|
||||
|
||||
|
|
@ -268,23 +257,14 @@ def test_context_window_suffix_stripped_for_cost_lookup():
|
|||
"""
|
||||
from litellm.llms.bedrock.common_utils import get_bedrock_base_model
|
||||
|
||||
assert (
|
||||
get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1[1m]")
|
||||
== "anthropic.claude-opus-4-6-v1"
|
||||
)
|
||||
assert (
|
||||
get_bedrock_base_model("us.anthropic.claude-sonnet-4-6[1m]")
|
||||
== "anthropic.claude-sonnet-4-6"
|
||||
)
|
||||
assert get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1[1m]") == "anthropic.claude-opus-4-6-v1"
|
||||
assert get_bedrock_base_model("us.anthropic.claude-sonnet-4-6[1m]") == "anthropic.claude-sonnet-4-6"
|
||||
assert (
|
||||
get_bedrock_base_model("global.anthropic.claude-opus-4-5-20251101-v1:0[1m]")
|
||||
== "anthropic.claude-opus-4-5-20251101-v1:0"
|
||||
)
|
||||
# Ensure models without suffix are unaffected
|
||||
assert (
|
||||
get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1")
|
||||
== "anthropic.claude-opus-4-6-v1"
|
||||
)
|
||||
assert get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1") == "anthropic.claude-opus-4-6-v1"
|
||||
# Ensure :51k throughput suffix still works
|
||||
assert (
|
||||
get_bedrock_base_model("anthropic.claude-3-5-sonnet-20241022-v2:0:51k")
|
||||
|
|
@ -334,9 +314,7 @@ def test_output_config_effort_normalization_uses_model_info_ceiling(monkeypatch)
|
|||
("us.anthropic.claude-opus-4-7", "xhigh"),
|
||||
],
|
||||
)
|
||||
def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling(
|
||||
model, expected_ceiling
|
||||
):
|
||||
def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling(model, expected_ceiling):
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
|
||||
model_info = GetModelCostMap.load_local_model_cost_map()[model]
|
||||
|
|
@ -355,54 +333,24 @@ def test_route_prefix_matched_as_path_segment_not_substring():
|
|||
or a ``/`` boundary.
|
||||
"""
|
||||
# The bedrock_mantle/ provider prefix must NOT be read as the mantle/ route.
|
||||
assert (
|
||||
BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.5") != "mantle"
|
||||
)
|
||||
assert (
|
||||
BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.4") == "converse"
|
||||
)
|
||||
assert (
|
||||
BedrockModelInfo._explicit_mantle_route("bedrock_mantle/openai.gpt-5.5")
|
||||
is False
|
||||
)
|
||||
assert BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.5") != "mantle"
|
||||
assert BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.4") == "converse"
|
||||
assert BedrockModelInfo._explicit_mantle_route("bedrock_mantle/openai.gpt-5.5") is False
|
||||
|
||||
# A genuine mantle route still resolves, via the startswith branch...
|
||||
assert (
|
||||
BedrockModelInfo.get_bedrock_route("mantle/anthropic.claude-mythos-preview")
|
||||
== "mantle"
|
||||
)
|
||||
assert BedrockModelInfo.get_bedrock_route("mantle/anthropic.claude-mythos-preview") == "mantle"
|
||||
# ...and via the mid-path "/mantle/" branch (after the bedrock/ provider prefix).
|
||||
assert (
|
||||
BedrockModelInfo.get_bedrock_route(
|
||||
"bedrock/mantle/anthropic.claude-mythos-preview"
|
||||
)
|
||||
== "mantle"
|
||||
)
|
||||
assert BedrockModelInfo.get_bedrock_route("bedrock/mantle/anthropic.claude-mythos-preview") == "mantle"
|
||||
|
||||
|
||||
def test_model_has_route_prefix_exercises_both_branches():
|
||||
"""``_model_has_route_prefix`` matches on ``startswith`` or a ``/`` boundary only."""
|
||||
# startswith branch
|
||||
assert (
|
||||
BedrockModelInfo._model_has_route_prefix(
|
||||
"mantle/anthropic.claude-mythos-preview", "mantle/"
|
||||
)
|
||||
is True
|
||||
)
|
||||
assert BedrockModelInfo._model_has_route_prefix("mantle/anthropic.claude-mythos-preview", "mantle/") is True
|
||||
# f"/{prefix}" boundary branch
|
||||
assert (
|
||||
BedrockModelInfo._model_has_route_prefix(
|
||||
"bedrock/mantle/anthropic.claude-mythos-preview", "mantle/"
|
||||
)
|
||||
is True
|
||||
)
|
||||
assert BedrockModelInfo._model_has_route_prefix("bedrock/mantle/anthropic.claude-mythos-preview", "mantle/") is True
|
||||
# neither branch: the token only appears glued to another segment
|
||||
assert (
|
||||
BedrockModelInfo._model_has_route_prefix(
|
||||
"bedrock_mantle/openai.gpt-5.5", "mantle/"
|
||||
)
|
||||
is False
|
||||
)
|
||||
assert BedrockModelInfo._model_has_route_prefix("bedrock_mantle/openai.gpt-5.5", "mantle/") is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -452,16 +400,10 @@ def test_explicit_invoke_route_does_not_match_async_invoke():
|
|||
"""
|
||||
async_invoke_model = "async_invoke/twelvelabs.marengo-embed-2-7-v1:0"
|
||||
assert BedrockModelInfo._explicit_invoke_route(async_invoke_model) is False
|
||||
assert (
|
||||
BedrockModelInfo._explicit_invoke_route(f"bedrock/{async_invoke_model}")
|
||||
is False
|
||||
)
|
||||
assert BedrockModelInfo._explicit_invoke_route(f"bedrock/{async_invoke_model}") is False
|
||||
# ...while async_invoke/ is still detected as its own route.
|
||||
assert BedrockModelInfo._explicit_async_invoke_route(async_invoke_model) is True
|
||||
assert (
|
||||
BedrockModelInfo._explicit_async_invoke_route(f"bedrock/{async_invoke_model}")
|
||||
is True
|
||||
)
|
||||
assert BedrockModelInfo._explicit_async_invoke_route(f"bedrock/{async_invoke_model}") is True
|
||||
|
||||
|
||||
def test_capability_lookups_fall_back_to_base_model_when_regional_entry_lacks_field(monkeypatch):
|
||||
|
|
@ -1285,6 +1227,7 @@ def _vcr_outcome_gate(request, vcr):
|
|||
yield
|
||||
record_vcr_outcome(request, vcr)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def event_loop():
|
||||
try:
|
||||
|
|
@ -1294,6 +1237,7 @@ def event_loop():
|
|||
yield loop
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def setup_and_teardown(event_loop):
|
||||
import litellm
|
||||
|
|
@ -1327,6 +1271,7 @@ def setup_and_teardown(event_loop):
|
|||
if pending:
|
||||
event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
|
||||
|
||||
|
||||
_SCALAR_DEFAULTS = {
|
||||
"num_retries": getattr(litellm, "num_retries", None),
|
||||
"set_verbose": getattr(litellm, "set_verbose", False),
|
||||
|
|
@ -1341,6 +1286,7 @@ _SCALAR_DEFAULTS = {
|
|||
"cohere_key": getattr(litellm, "cohere_key", None),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
class TestStripBedrockRoutingPrefix:
|
||||
"""Tests for strip_bedrock_routing_prefix function."""
|
||||
|
|
@ -1371,6 +1317,7 @@ class TestStripBedrockRoutingPrefix:
|
|||
== "anthropic.claude-3-sonnet-20240229-v1:0"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
class TestStripBedrockThroughputSuffix:
|
||||
"""Tests for strip_bedrock_throughput_suffix function."""
|
||||
|
|
@ -1398,6 +1345,7 @@ class TestStripBedrockThroughputSuffix:
|
|||
def test_strip_throughput_suffix(self, input_model, expected):
|
||||
assert strip_bedrock_throughput_suffix(input_model) == expected
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
class TestExtractModelNameFromBedrockArn:
|
||||
"""Tests for extract_model_name_from_bedrock_arn function."""
|
||||
|
|
@ -1418,6 +1366,7 @@ class TestExtractModelNameFromBedrockArn:
|
|||
arn = "ARN:aws:bedrock:us-east-1:123456789012:model/my-model"
|
||||
assert extract_model_name_from_bedrock_arn(arn) == "my-model"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
class TestGetBedrockCrossRegionInferenceRegions:
|
||||
"""Tests for get_bedrock_cross_region_inference_regions function."""
|
||||
|
|
@ -1433,6 +1382,7 @@ class TestGetBedrockCrossRegionInferenceRegions:
|
|||
regions = get_bedrock_cross_region_inference_regions()
|
||||
assert isinstance(regions, list)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
class TestGetBedrockBaseModel:
|
||||
"""Tests for get_bedrock_base_model function."""
|
||||
|
|
@ -1496,6 +1446,7 @@ class TestGetBedrockBaseModel:
|
|||
"""Test that throughput tier suffixes like :51k are stripped. Issue #19113."""
|
||||
assert get_bedrock_base_model(input_model) == expected
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
class TestBedrockModelInfoWrappers:
|
||||
"""Tests that BedrockModelInfo methods correctly wrap standalone functions."""
|
||||
|
|
@ -1517,6 +1468,7 @@ class TestBedrockModelInfoWrappers:
|
|||
model = "bedrock/converse/claude-3"
|
||||
assert BedrockModelInfo.get_non_litellm_routing_model_name(model) == strip_bedrock_routing_prefix(model)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
class TestBedrockTokenCounter:
|
||||
"""Tests for BedrockTokenCounter class."""
|
||||
|
|
@ -1553,12 +1505,14 @@ class TestBedrockTokenCounter:
|
|||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _pr4_bedrock_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "pr4-test-aws-access-key")
|
||||
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "pr4-test-aws-secret-key")
|
||||
monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _use_local_model_cost_map() -> Iterator[None]:
|
||||
with pytest.MonkeyPatch.context() as monkeypatch:
|
||||
|
|
@ -1566,6 +1520,7 @@ def _use_local_model_cost_map() -> Iterator[None]:
|
|||
importlib.reload(litellm)
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(
|
||||
"_pr4_bedrock_env",
|
||||
"_use_local_model_cost_map",
|
||||
|
|
@ -2106,6 +2061,7 @@ class TestBedrockGovCloudSupport:
|
|||
@pytest.mark.parametrize("missing", ["botocore", "unrelated_dependency"])
|
||||
def test_event_decoder_reports_only_missing_aws_dependency(missing):
|
||||
from unittest.mock import patch
|
||||
|
||||
from litellm.llms.bedrock.common_utils import BedrockEventStreamDecoderBase
|
||||
|
||||
failure = ModuleNotFoundError("missing dependency", name=missing)
|
||||
|
|
@ -2142,8 +2098,12 @@ def test_bedrock_capability_model_reads_base_model_only_for_application_profile_
|
|||
assert bedrock_capability_model(model, base_model) == expected
|
||||
|
||||
|
||||
_FORMAT_REJECTION: Final = "The model returned the following errors: output_config.format: Extra inputs are not permitted"
|
||||
_COMPILE_REJECTION: Final = "The model returned the following errors: compilation exceeded limits: Schema is too complex."
|
||||
_FORMAT_REJECTION: Final = (
|
||||
"The model returned the following errors: output_config.format: Extra inputs are not permitted"
|
||||
)
|
||||
_COMPILE_REJECTION: Final = (
|
||||
"The model returned the following errors: compilation exceeded limits: Schema is too complex."
|
||||
)
|
||||
_SCHEMA_PATTERN_REJECTION: Final = (
|
||||
"The model returned the following errors: output_config.format.schema: Unsupported regex feature in pattern "
|
||||
"field: pattern is too complex for structured output: reduce the {n,m} upper bound, narrow the character class "
|
||||
|
|
@ -2244,7 +2204,10 @@ def test_bedrock_strict_tools_fallback_drops_only_strict_and_keeps_everything_el
|
|||
assert fallback == NativeOutputFallback(
|
||||
optional_params={
|
||||
"tools": [
|
||||
{**strict_tool, "function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}}},
|
||||
{
|
||||
**strict_tool,
|
||||
"function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}},
|
||||
},
|
||||
plain_tool,
|
||||
anthropic_tool,
|
||||
{"toolSpec": {"name": "spec", "inputSchema": {"json": {"type": "object"}}}},
|
||||
|
|
@ -2299,3 +2262,45 @@ def test_bedrock_strict_tools_fallback_retries_only_when_the_request_sent_strict
|
|||
fallback: Final = bedrock_strict_tools_fallback(model, {"tools": tools}, litellm_params)
|
||||
|
||||
assert (fallback is not None) == sent_strict
|
||||
|
||||
|
||||
_TIER_PRICED_ROW: Final = {
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"input_cost_per_token": 1e-6,
|
||||
"input_cost_per_token_priority": 2e-6,
|
||||
"input_cost_per_token_flex": 5e-7,
|
||||
}
|
||||
_STANDARD_ONLY_ROW: Final = {
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"input_cost_per_token": 1e-6,
|
||||
"input_cost_per_token_priority": None,
|
||||
"input_cost_per_token_flex": None,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "service_tier", "expected"),
|
||||
[
|
||||
("tiered.model-v1:0", "priority", True),
|
||||
("tiered.model-v1:0", "flex", True),
|
||||
("bedrock/us-west-2/tiered.model-v1:0", "flex", True),
|
||||
("eu.tiered.model-v1:0", "priority", True),
|
||||
("standard.model-v1:0", "priority", False),
|
||||
("standard.model-v1:0", "flex", False),
|
||||
("standard.model-v1:0", "default", True),
|
||||
("standard.model-v1:0", "auto", True),
|
||||
("a-model-the-price-map-has-never-heard-of", "priority", True),
|
||||
],
|
||||
)
|
||||
def test_bedrock_serves_service_tier_follows_price_map_tier_rows(
|
||||
monkeypatch: pytest.MonkeyPatch, model: str, service_tier: str, expected: bool
|
||||
) -> None:
|
||||
"""A Bedrock model serves a tier exactly when its price-map row carries a non-null price for that tier.
|
||||
The proxy expands every row to the full ModelInfo shape, so a null price (standard-only row) is no signal,
|
||||
`default` and `auto` are always served, and an unknown model keeps being forwarded. A regional CRIS row
|
||||
that omits the tier prices does not hide the base row that carries them."""
|
||||
monkeypatch.setitem(litellm.model_cost, "tiered.model-v1:0", dict(_TIER_PRICED_ROW))
|
||||
monkeypatch.setitem(litellm.model_cost, "eu.tiered.model-v1:0", dict(_STANDARD_ONLY_ROW))
|
||||
monkeypatch.setitem(litellm.model_cost, "standard.model-v1:0", dict(_STANDARD_ONLY_ROW))
|
||||
|
||||
assert bedrock_serves_service_tier(model, service_tier) is expected
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue