mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-01 07:03:02 +00:00
fix(bedrock): route a Bedrock model id with no Invoke provider to Converse (#43290)
* fix(bedrock): route unmapped converse-only family model ids to converse #42713 stopped Bedrock-native OpenAI ids that are not in the cost map yet from falling to the invoke route. The same gap exists for the other families that have no invoke transformation at all: minimax, nvidia, writer, zai, google and xai. An id from one of them that is newer than the cost map hits get_bedrock_route -> "invoke" -> get_bedrock_invoke_provider() -> None -> BedrockError, when Converse would have served it Add a second family regex for those six and default them to converse, the same way the openai family is handled. _OPENAI_FAMILY_MODEL_RE is left alone because the prompt-caching rule is openai-specific Every shipped id of these families already resolves to a converse entry via its base model (checked against the cost map), so this only changes behaviour for ids that are not in the map yet Tests: six unmapped ids route to converse, and an unmapped meta.* id still routes to invoke since meta has an invoke handler * fix(bedrock): route a model id with no Invoke provider to Converse A bedrock/<model> id missing from the cost map fell to the Invoke route, where an id that names no Invoke provider (a new vendor's model, a bare provisioned-model, imported-model, or custom-model-deployment ARN) fails locally with "Unknown provider=None" before any request is sent. Such ids now go to Converse, which replaces the per-family regex with the general rule. Ids that name an Invoke provider and explicit route prefixes are unchanged. * test(bedrock): expect Converse for ids with no Invoke provider in the nova ARN and not-found tests * fix(bedrock): accept tools on Converse for a model with no cost-map row or Invoke provider, and keep a 404 for an invalid id * fix(bedrock): answer 404 for an invalid model id instead of AWS's 400 AWS answers a mistyped Bedrock model id with a 400 ValidationException. The mapping already raised NotFoundError, but passing AWS's 400 response through made the openai base class report 400, so the proxy answered 400 and the router skipped its cooldown * fix(bedrock): drop lookaround regexes from tool schemas for a model the cost map does not list * fix(bedrock): apply the unknown-model defaults only to bare ids LiteLLM knows nothing about A bare Bedrock id with no cost-map row and no Invoke vendor gets Converse's open params, its reasoning_effort as sent, lookaround-free tool schemas, and a 404 for AWS's invalid-identifier reply, which keeps AWS's response headers. An explicit route like converse/, an ARN, or an id LiteLLM recognizes keeps main's params, regexes and error status, so drop_params still drops tools there * fix(bedrock): judge lookaround support by model id on every route, ignore price-only rows The regex check now strips route and region prefixes from the model and its base_model before asking whether LiteLLM knows the model, so bedrock/converse/<id> gets the same answer in a direct transform call as on the completion path. A cost-map row counts only when it names a provider, so a price-only register_model row leaves an unlisted id unknown, as on the proxy. AmazonInvokeConfig now inherits the Invoke provider lookup instead of keeping an identical copy. * fix(bedrock): treat a region path like the bare id when the cost map does not list it A deployment like bedrock/us-east-1/<unlisted id> now gets the same tools, reasoning_effort and typo 404 as bedrock/<unlisted id>, so Claude Code works on it and a mistyped one still answers 404. * fix(bedrock): keep today's 404 for ids that reach Converse only by default, and keep an empty id local * fix(bedrock): answer 404 for an invalid id only when Converse rejected it, and keep a region path with no id local * fix(bedrock): keep a path-shaped id local, and map /v1/completions errors with the model's provider An id with a slash that is not an ARN, like bedrock/../../converse or an unknown region path, keeps Invoke and its local 404 instead of reaching AWS Converse. atext_completion mapped every error as OpenAI, so a Bedrock typo on /v1/completions answered 400 while chat answered 404. It now resolves the provider from the request's model, as the other async entry points do. * fix(text completion): keep /v1/completions error mapping as it is on main Mapping async /v1/completions errors with each provider's rules is wider than this Bedrock routing change. It moves classes, statuses, cooldown and fallbacks for most providers, and some lose context-window fallbacks. So this restores main's mapping here and leaves that change to its own PR. A Bedrock typo on /v1/completions answers AWS's 400, while chat keeps the 404. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
Siddhant Badola
GitHub
parent
a44e44b980
commit
b71491c6c9
9 changed files with 559 additions and 108 deletions
|
|
@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import (
|
|||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
|
||||
from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
|
||||
from litellm.llms.bedrock.common_utils import bedrock_model_reaches_converse_by_default
|
||||
from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
|
|
@ -938,13 +939,22 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE: Final = (
|
|||
)
|
||||
|
||||
|
||||
def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPException) -> httpx.Response:
|
||||
def _bedrock_response_with_status(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response:
|
||||
original_response: Final = getattr(original_exception, "response", None)
|
||||
if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None:
|
||||
return httpx.Response(status_code=400, request=httpx.Request(method="POST", url="https://api.openai.com/v1/"))
|
||||
if original_response.status_code == 400:
|
||||
return httpx.Response(
|
||||
status_code=status_code, request=httpx.Request(method="POST", url="https://api.openai.com/v1/")
|
||||
)
|
||||
if original_response.status_code == status_code:
|
||||
return original_response
|
||||
return httpx.Response(status_code=400, headers=original_response.headers, request=original_response.request)
|
||||
return httpx.Response(status_code=status_code, headers=original_response.headers, request=original_response.request)
|
||||
|
||||
|
||||
def _came_from_bedrock_converse(original_exception: _ProviderHTTPException) -> bool:
|
||||
original_response: Final = getattr(original_exception, "response", None)
|
||||
if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None:
|
||||
return False
|
||||
return original_response.request.url.path.endswith(("/converse", "/converse-stream"))
|
||||
|
||||
|
||||
def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
|
||||
|
|
@ -980,7 +990,7 @@ def _map_bedrock_exception(
|
|||
message=mantle_context_window_message,
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
response=_bedrock_mantle_context_window_response(original_exception),
|
||||
response=_bedrock_response_with_status(original_exception, 400),
|
||||
)
|
||||
if (
|
||||
"too many tokens" in error_str
|
||||
|
|
@ -1003,6 +1013,18 @@ def _map_bedrock_exception(
|
|||
llm_provider="bedrock",
|
||||
response=getattr(original_exception, "response", None),
|
||||
)
|
||||
elif (
|
||||
"The provided model identifier is invalid" in error_str
|
||||
and bool(model)
|
||||
and _came_from_bedrock_converse(original_exception)
|
||||
and bedrock_model_reaches_converse_by_default(model)
|
||||
):
|
||||
raise NotFoundError(
|
||||
message=f"BedrockException - {error_str}",
|
||||
model=model,
|
||||
llm_provider="bedrock",
|
||||
response=_bedrock_response_with_status(original_exception, 404),
|
||||
)
|
||||
elif "Malformed input request" in error_str:
|
||||
raise BadRequestError(
|
||||
message=f"BedrockException - {error_str}",
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_image
|
|||
from litellm.llms.bedrock.common_utils import (
|
||||
bedrock_chat_uses_native_structured_output,
|
||||
bedrock_cleaned_output_schema,
|
||||
bedrock_model_is_unknown,
|
||||
bedrock_model_supports_regex_lookaround,
|
||||
bedrock_native_output_format,
|
||||
bedrock_native_output_is_opt_in,
|
||||
|
|
@ -529,13 +530,14 @@ class AmazonConverseConfig(BaseConfig):
|
|||
"""
|
||||
Handle the reasoning_effort parameter based on the model type.
|
||||
|
||||
- GPT-OSS and DeepSeek V3 models: passed through unchanged via additionalModelRequestFields.
|
||||
- GPT-OSS, DeepSeek V3 and unknown models (``bedrock_model_is_unknown``): passed through unchanged
|
||||
via additionalModelRequestFields.
|
||||
- OpenAI GPT-5.x and GPT-6 models: mapped to ``reasoning.effort`` via additionalModelRequestFields.
|
||||
- Nova 2 models: transformed to reasoningConfig.
|
||||
- Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on
|
||||
adaptive Claude 4.6 / 4.7).
|
||||
"""
|
||||
if "gpt-oss" in model or "deepseek" in model:
|
||||
if "gpt-oss" in model or "deepseek" in model or bedrock_model_is_unknown(model):
|
||||
optional_params["reasoning_effort"] = reasoning_effort
|
||||
elif self._is_openai_gpt_reasoning_model(model):
|
||||
reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort}
|
||||
|
|
@ -681,7 +683,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
"parallel_tool_calls",
|
||||
]
|
||||
|
||||
if "arn" in model: # we can't infer the model from the arn, so just add all params
|
||||
if "arn" in model or bedrock_model_is_unknown(model):
|
||||
supported_params.append("tools")
|
||||
supported_params.append("tool_choice")
|
||||
supported_params.append("thinking")
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import copy
|
|||
import json
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, cast, get_args
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
|
||||
|
|
@ -74,10 +74,6 @@ _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
|
|||
"{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
|
||||
)
|
||||
|
||||
_INVOKE_PROVIDERS: Final = TypeAdapter(tuple[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, ...]).validate_python(
|
||||
get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL)
|
||||
)
|
||||
|
||||
|
||||
def _response_header(headers: Mapping[str, str], name: str) -> str | None:
|
||||
return headers.get(name, None)
|
||||
|
|
@ -562,62 +558,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
"""
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_bedrock_invoke_provider(
|
||||
model: str,
|
||||
) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
|
||||
"""
|
||||
Helper function to get the bedrock provider from the model
|
||||
|
||||
handles 4 scenarios:
|
||||
1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
|
||||
2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
|
||||
3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama`
|
||||
4. model=us.amazon.nova-pro-v1:0 -> Returns `nova`
|
||||
"""
|
||||
if model.startswith("invoke/"):
|
||||
model = model.replace("invoke/", "", 1)
|
||||
|
||||
# Special case: Check for "nova" in model name first (before "amazon")
|
||||
# This handles amazon.nova-* models which would otherwise match "amazon" (Titan)
|
||||
if "nova" in model.lower():
|
||||
if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
||||
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova")
|
||||
|
||||
_split_model: Final = model.split(".")[0]
|
||||
if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
||||
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model)
|
||||
|
||||
# If not a known provider, check for pattern with two slashes
|
||||
provider = AmazonInvokeConfig._get_provider_from_model_path(model)
|
||||
if provider is not None:
|
||||
return provider
|
||||
|
||||
for provider in _INVOKE_PROVIDERS:
|
||||
if provider in model:
|
||||
return provider
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_provider_from_model_path(
|
||||
model_path: str,
|
||||
) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
|
||||
"""
|
||||
Helper function to get the provider from a model path with format: provider/model-name
|
||||
|
||||
Args:
|
||||
model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name')
|
||||
|
||||
Returns:
|
||||
Optional[str]: The provider name, or None if no valid provider found
|
||||
"""
|
||||
parts: Final = model_path.split("/")
|
||||
if len(parts) >= 1:
|
||||
provider: Final = parts[0]
|
||||
if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
||||
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider)
|
||||
return None
|
||||
|
||||
def convert_messages_to_prompt(self, model: str, messages, provider, custom_prompt_dict) -> tuple[str, list | None]:
|
||||
# handle anthropic prompts and amazon titan prompts
|
||||
prompt = ""
|
||||
|
|
|
|||
|
|
@ -1098,6 +1098,11 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None:
|
|||
return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw)
|
||||
|
||||
|
||||
def _cost_map_row_names_a_provider(key: str) -> bool:
|
||||
entry: Final = _model_cost_entry(key)
|
||||
return entry is not None and entry.get("litellm_provider") is not None
|
||||
|
||||
|
||||
def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]:
|
||||
return tuple(
|
||||
_model_cost_entry(key)
|
||||
|
|
@ -1567,7 +1572,8 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
|
|||
|
||||
The deployment's ``model_info.supports_regex_lookaround`` wins, then the
|
||||
``model_prices_and_context_window.json`` entry of its ``base_model``, then the
|
||||
entry of ``model`` itself. A model nobody flagged keeps its schema as sent.
|
||||
entry of ``model`` itself. A model nobody flagged keeps its schema as sent, unless
|
||||
no candidate names a model LiteLLM knows, whatever route prefix it carries.
|
||||
"""
|
||||
params: Final = litellm_params or {}
|
||||
deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround")
|
||||
|
|
@ -1576,7 +1582,41 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
|
|||
base_model: Final = params.get("base_model")
|
||||
candidates: Final = (*((base_model,) if isinstance(base_model, str) else ()), model)
|
||||
flags: Final = (_bedrock_converse_model_flag(candidate, "supports_regex_lookaround") for candidate in candidates)
|
||||
return next((flag for flag in flags if flag is not None), True)
|
||||
flag: Final = next((flag for flag in flags if flag is not None), None)
|
||||
if flag is not None:
|
||||
return flag
|
||||
return not all(bedrock_model_is_unknown(split_bedrock_region_path(candidate)[1]) for candidate in candidates)
|
||||
|
||||
|
||||
def _bedrock_model_id_without_region(model: str) -> str:
|
||||
routed_id: Final = model.removeprefix("bedrock/")
|
||||
region, separator, regional_id = routed_id.partition("/")
|
||||
return regional_id if separator and region in get_all_bedrock_regions() else routed_id
|
||||
|
||||
|
||||
def bedrock_model_is_unknown(model: str) -> bool:
|
||||
model_id: Final = _bedrock_model_id_without_region(model)
|
||||
if "/" in model_id:
|
||||
return False
|
||||
base: Final = get_bedrock_base_model(model_id)
|
||||
regional_ids: Final = (f"{region}.{base}" for region in get_bedrock_cross_region_inference_regions())
|
||||
if any(_cost_map_row_names_a_provider(key) for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)):
|
||||
return False
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
|
||||
return BaseAWSLLM.get_bedrock_invoke_provider(model_id) is None
|
||||
|
||||
|
||||
def _names_no_invoke_provider(model: str) -> bool:
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
|
||||
model_id: Final = _bedrock_model_id_without_region(model)
|
||||
names_one_model: Final = bool(model_id) and ("/" not in model_id or model_id.startswith("arn:"))
|
||||
return names_one_model and BaseAWSLLM.get_bedrock_invoke_provider(model) is None
|
||||
|
||||
|
||||
def bedrock_model_reaches_converse_by_default(model: str) -> bool:
|
||||
return BedrockModelInfo.named_bedrock_route(model, None) is None and _names_no_invoke_provider(model)
|
||||
|
||||
|
||||
_BedrockConverseModelFlag: TypeAlias = Literal[
|
||||
|
|
@ -1585,10 +1625,13 @@ _BedrockConverseModelFlag: TypeAlias = Literal[
|
|||
]
|
||||
|
||||
|
||||
def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None:
|
||||
def _bedrock_model_info_candidates(model: str) -> tuple[str, ...]:
|
||||
base: Final = get_bedrock_base_model(model)
|
||||
candidates: Final = dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base)))
|
||||
for candidate in candidates:
|
||||
return tuple(dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base))))
|
||||
|
||||
|
||||
def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None:
|
||||
for candidate in _bedrock_model_info_candidates(model):
|
||||
with contextlib.suppress(Exception):
|
||||
model_info = get_cached_model_info()(
|
||||
model=candidate,
|
||||
|
|
@ -1766,8 +1809,19 @@ class BedrockModelInfo(BaseLLMModelInfo):
|
|||
default (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in;
|
||||
``request_params`` (the caller's chat params) sends such a request to Converse when it needs a
|
||||
feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family
|
||||
model stays on Converse without the prefix.
|
||||
model stays on Converse without the prefix. A model the cost map does not list as Converse goes to
|
||||
Invoke only when its id names an Invoke provider, and to Converse otherwise.
|
||||
"""
|
||||
named_route: Final = BedrockModelInfo.named_bedrock_route(model, request_params)
|
||||
if named_route is not None:
|
||||
return named_route
|
||||
return "converse" if _names_no_invoke_provider(model) else "invoke"
|
||||
|
||||
@staticmethod
|
||||
def named_bedrock_route(
|
||||
model: str,
|
||||
request_params: Mapping[str, object] | None,
|
||||
) -> BedrockRoute | None:
|
||||
route_mappings: dict[
|
||||
str,
|
||||
Literal[
|
||||
|
|
@ -1822,7 +1876,7 @@ class BedrockModelInfo(BaseLLMModelInfo):
|
|||
return "converse"
|
||||
if _OPENAI_FAMILY_MODEL_RE.search(base_model):
|
||||
return "converse"
|
||||
return "invoke"
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _explicit_converse_route(model: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -1037,7 +1037,7 @@ def test_s07_a_5kb_caller_model_under_the_wildcard_is_a_bounded_4xx_and_liveline
|
|||
error: Final = _JSON.validate_python(_JSON.validate_python(json.loads(outcome.raw))["error"])
|
||||
assert "a" * 5120 in str(error["message"]), outcome.raw[:300]
|
||||
assert_within_factor_of_control(oversized.seconds, control=control.seconds, factor=20)
|
||||
assert rig.wire.drain() == ()
|
||||
assert len(rig.wire.drain()) == 1
|
||||
_failure_row(outcome.call_id)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -60,7 +60,6 @@ def test_openai_cyber_policy_uses_the_structured_error_code(shape: str, code: st
|
|||
assert caught.value.response.headers["x-request-id"] == "cyber-policy-request"
|
||||
|
||||
|
||||
|
||||
def _debug_info_for_api_base(configured_api_base: str) -> str:
|
||||
request: Final = httpx.Request("POST", "https://gateway.example/v1/chat/completions")
|
||||
original: Final = OpenAIError(
|
||||
|
|
@ -1602,6 +1601,75 @@ def test_bedrock_mantle_400_maps_to_bad_request():
|
|||
assert type(excinfo.value) is litellm.BadRequestError
|
||||
|
||||
|
||||
class _ErrorWithoutResponse(Exception):
|
||||
def __init__(self, message: str, status_code: int) -> None:
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
def test_bedrock_mantle_context_overflow_without_a_provider_response_answers_400():
|
||||
with pytest.raises(litellm.ContextWindowExceededError) as excinfo:
|
||||
exception_type(
|
||||
model="openai.gpt-5.6-sol",
|
||||
original_exception=_ErrorWithoutResponse(
|
||||
'{"error":{"code":"context_length_exceeded","message":"too long"}}', 400
|
||||
),
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
)
|
||||
|
||||
assert excinfo.value.response.status_code == 400
|
||||
|
||||
|
||||
def _invalid_model_identifier_from(url: str) -> BedrockError:
|
||||
return BedrockError(
|
||||
status_code=400,
|
||||
message='{"message":"The provided model identifier is invalid."}',
|
||||
response=httpx.Response(status_code=400, request=httpx.Request(method="POST", url=url)),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path",
|
||||
["/model/us.newvendor.typo-1/converse", "/model/us.newvendor.typo-1/converse-stream"],
|
||||
)
|
||||
def test_bedrock_converse_rejecting_an_id_with_no_invoke_provider_answers_404(path):
|
||||
with pytest.raises(litellm.NotFoundError) as excinfo:
|
||||
exception_type(
|
||||
model="us.newvendor.typo-1",
|
||||
original_exception=_invalid_model_identifier_from(f"https://bedrock-runtime.us-east-1.amazonaws.com{path}"),
|
||||
custom_llm_provider="bedrock",
|
||||
)
|
||||
|
||||
assert excinfo.value.response.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"original_exception",
|
||||
[
|
||||
_invalid_model_identifier_from("https://bedrock-runtime.us-east-1.amazonaws.com/model/claude-typo-9/invoke"),
|
||||
_ErrorWithoutResponse("The provided model identifier is invalid.", 400),
|
||||
],
|
||||
ids=["invoke", "no-response"],
|
||||
)
|
||||
def test_bedrock_invalid_model_identifier_not_from_converse_stays_a_bad_request(original_exception):
|
||||
with pytest.raises(litellm.BadRequestError) as excinfo:
|
||||
exception_type(model="claude-typo-9", original_exception=original_exception, custom_llm_provider="bedrock")
|
||||
|
||||
assert excinfo.value.status_code == 400
|
||||
|
||||
|
||||
def test_bedrock_invalid_model_identifier_without_a_model_stays_a_bad_request():
|
||||
with pytest.raises(litellm.BadRequestError) as excinfo:
|
||||
exception_type(
|
||||
model=None,
|
||||
original_exception=_ErrorWithoutResponse("The provided model identifier is invalid.", 400),
|
||||
custom_llm_provider="bedrock",
|
||||
)
|
||||
|
||||
assert excinfo.value.status_code == 400
|
||||
|
||||
|
||||
def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded():
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
|
|
|
|||
|
|
@ -1221,6 +1221,97 @@ def test_transform_request_drops_lookaround_regex_when_the_deployment_model_info
|
|||
assert properties == _LOOKAROUND_FREE_PROPERTIES
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "litellm_params"),
|
||||
[
|
||||
("us.newvendor.model-1", None),
|
||||
("converse/us.newvendor.model-1", None),
|
||||
("us-east-1/us.newvendor.model-1", None),
|
||||
("us.newvendor.model-1", {"base_model": "bedrock/converse/us.newvendor.model-1"}),
|
||||
],
|
||||
)
|
||||
def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list(model, litellm_params):
|
||||
assert "us.newvendor.model-1" not in litellm.model_cost
|
||||
|
||||
properties: Final = _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL, litellm_params)
|
||||
|
||||
assert properties == _LOOKAROUND_FREE_PROPERTIES
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"arn:aws:bedrock:us-east-1:123456789012:custom-model-deployment/abc123",
|
||||
"arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123",
|
||||
"us.anthropic.claude-unreleased-v1:0",
|
||||
"eu.openai.gpt-unreleased",
|
||||
"apac.newvendor.listed-1",
|
||||
],
|
||||
)
|
||||
def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_litellm_still_recognizes(model, monkeypatch):
|
||||
monkeypatch.setitem(litellm.model_cost, "us.newvendor.listed-1", {"litellm_provider": "bedrock_converse"})
|
||||
|
||||
properties = _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL)
|
||||
|
||||
assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"]
|
||||
|
||||
|
||||
def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_whose_deployment_opts_out():
|
||||
properties = _tool_schema_properties(
|
||||
"us.newvendor.model-1", _ARTIFACT_DATA_ANTHROPIC_TOOL, {"model_info": {"supports_regex_lookaround": True}}
|
||||
)
|
||||
|
||||
assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployment_params",
|
||||
[
|
||||
{"model": "bedrock/us.newvendor.model-1"},
|
||||
{"model": "bedrock/converse/us.newvendor.model-1", "allowed_openai_params": ["tools"]},
|
||||
],
|
||||
)
|
||||
def test_a_router_deployment_of_a_model_the_cost_map_does_not_list_drops_lookaround_regex(
|
||||
respx_mock, deployment_params
|
||||
):
|
||||
route: Final = respx_mock.post(
|
||||
"https://bedrock-runtime.us-east-1.amazonaws.com/model/us.newvendor.model-1/converse"
|
||||
).mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2},
|
||||
},
|
||||
)
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "unlisted",
|
||||
"litellm_params": {
|
||||
**deployment_params,
|
||||
"aws_access_key_id": "fake-access-key",
|
||||
"aws_secret_access_key": "fake-secret-key",
|
||||
"aws_region_name": "us-east-1",
|
||||
},
|
||||
"model_info": {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
router.completion(
|
||||
model="unlisted",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=[_ARTIFACT_DATA_OPENAI_TOOL],
|
||||
client=HTTPHandler(),
|
||||
)
|
||||
|
||||
tool: Final = json.loads(route.calls[0].request.content)["toolConfig"]["tools"][0]
|
||||
assert tool["toolSpec"]["inputSchema"]["json"]["properties"] == _LOOKAROUND_FREE_PROPERTIES
|
||||
|
||||
|
||||
def test_transform_request_keeps_lookaround_regex_when_the_deployment_model_info_opts_out():
|
||||
properties = _tool_schema_properties(
|
||||
"global.moonshotai.kimi-k3", _ARTIFACT_DATA_ANTHROPIC_TOOL, {"model_info": {"supports_regex_lookaround": True}}
|
||||
|
|
@ -9189,18 +9280,223 @@ def test_bedrock_tools_transformation_valid_params():
|
|||
assert "test" in result[0]["toolSpec"]["inputSchema"]["json"]["required"]
|
||||
|
||||
|
||||
def test_not_found_error():
|
||||
def _invalid_model_identifier_reply(sent: list[httpx.Request]):
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
sent.append(request)
|
||||
return httpx.Response(
|
||||
400,
|
||||
json={"message": "The provided model identifier is invalid."},
|
||||
headers={"x-amzn-ErrorType": "ValidationException", "x-amzn-RequestId": "req-123"},
|
||||
)
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, path",
|
||||
[
|
||||
("bedrock/bad_model", b"/model/bad_model/converse"),
|
||||
("bedrock/us-east-1/bad_model", b"/model/bad_model/converse"),
|
||||
("bedrock/newvendor.listed-v1:0", b"/model/newvendor.listed-v1%3A0/converse"),
|
||||
("bedrock/us-east-1/newvendor.listed-v1:0", b"/model/newvendor.listed-v1%3A0/converse"),
|
||||
],
|
||||
)
|
||||
def test_not_found_error(model, path, monkeypatch):
|
||||
monkeypatch.setitem(litellm.model_cost, "newvendor.listed-v1:0", {"litellm_provider": "bedrock", "mode": "chat"})
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
with pytest.raises(litellm.NotFoundError) as exc_info:
|
||||
completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "What is the meaning of life"}],
|
||||
client=HTTPHandler(
|
||||
client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent)))
|
||||
),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 404
|
||||
assert exc_info.value.response.status_code == 404
|
||||
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-123"
|
||||
assert [request.url.raw_path for request in sent] == [path]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model", ["bedrock/", "bedrock/us-east-1/", "bedrock/../../converse", "bedrock/xx-fake-1/us.xai.grok-4.7"]
|
||||
)
|
||||
def test_a_model_id_that_is_empty_or_a_path_fails_locally_without_calling_aws(model):
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
with pytest.raises(litellm.NotFoundError):
|
||||
completion(
|
||||
model="bedrock/bad_model",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What is the meaning of life",
|
||||
}
|
||||
],
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "What is the meaning of life"}],
|
||||
client=HTTPHandler(
|
||||
client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent)))
|
||||
),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert sent == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, path",
|
||||
[
|
||||
("bedrock/converse/bad_model", b"/model/bad_model/converse"),
|
||||
("bedrock/anthropic.claude-typo-v1:0", b"/model/anthropic.claude-typo-v1:0/invoke"),
|
||||
],
|
||||
)
|
||||
def test_invalid_model_identifier_stays_a_bad_request_for_a_model_litellm_recognizes(model, path):
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
with pytest.raises(litellm.BadRequestError) as exc_info:
|
||||
completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "What is the meaning of life"}],
|
||||
client=HTTPHandler(
|
||||
client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent)))
|
||||
),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert [request.url.raw_path for request in sent] == [path]
|
||||
|
||||
|
||||
_WEATHER_TOOL: Final = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather for a city",
|
||||
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _converse_tool_call_reply(sent: list[httpx.Request]):
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
sent.append(request)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"output": {
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"toolUse": {"toolUseId": "t1", "name": "get_weather", "input": {"city": "Paris"}}}
|
||||
],
|
||||
}
|
||||
},
|
||||
"stopReason": "tool_use",
|
||||
"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2},
|
||||
},
|
||||
)
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["bedrock/us.newvendor.model-1", "bedrock/us-east-1/us.newvendor.model-1"])
|
||||
def test_model_missing_from_cost_map_sends_tools_to_converse(model):
|
||||
assert "us.newvendor.model-1" not in litellm.model_cost
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
response: Final = completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
tools=[_WEATHER_TOOL],
|
||||
tool_choice="auto",
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert response.choices[0].message.tool_calls[0].function.name == "get_weather"
|
||||
assert [request.url.raw_path for request in sent] == [b"/model/us.newvendor.model-1/converse"]
|
||||
tool_config: Final = json.loads(sent[0].content)["toolConfig"]
|
||||
assert [tool["toolSpec"]["name"] for tool in tool_config["tools"]] == ["get_weather"]
|
||||
assert tool_config["toolChoice"] == {"auto": {}}
|
||||
|
||||
|
||||
def test_a_price_only_row_for_an_unlisted_model_still_sends_tools_to_converse(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost, "us.newvendor.priced-1", {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}
|
||||
)
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
completion(
|
||||
model="bedrock/us.newvendor.priced-1",
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
tools=[_WEATHER_TOOL],
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
tool_config: Final = json.loads(sent[0].content)["toolConfig"]
|
||||
assert [tool["toolSpec"]["name"] for tool in tool_config["tools"]] == ["get_weather"]
|
||||
|
||||
|
||||
def test_explicit_converse_route_to_an_unlisted_model_still_drops_tools_under_drop_params():
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
completion(
|
||||
model="bedrock/converse/us.newvendor.model-1",
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
tools=[_WEATHER_TOOL],
|
||||
drop_params=True,
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert "toolConfig" not in json.loads(sent[0].content)
|
||||
|
||||
|
||||
def test_unknown_model_gets_reasoning_effort_as_sent():
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
completion(
|
||||
model="bedrock/us.newvendor.model-1",
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
reasoning_effort="high",
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert json.loads(sent[0].content)["additionalModelRequestFields"] == {"reasoning_effort": "high"}
|
||||
|
||||
|
||||
def test_mapped_model_without_function_calling_still_rejects_tools():
|
||||
model: Final = "bedrock/converse/meta.llama3-8b-instruct-v1:0"
|
||||
assert "meta.llama3-8b-instruct-v1:0" in litellm.model_cost
|
||||
assert not litellm.supports_function_calling(model=model)
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError):
|
||||
completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
tools=[_WEATHER_TOOL],
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert sent == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, expected_base_model",
|
||||
|
|
|
|||
|
|
@ -214,24 +214,17 @@ def test_system_defined_inference_profile_arn_still_routes_to_converse():
|
|||
assert route == "converse"
|
||||
|
||||
|
||||
def test_other_opaque_arn_types_still_route_to_invoke():
|
||||
"""
|
||||
Only application-inference-profile ARNs are auto-routed to converse. Other
|
||||
opaque ARNs (provisioned-model, imported-model, custom-model-deployment)
|
||||
also yield no invoke provider, but they are frequently invoke-only with
|
||||
provider-specific body formats, so routing them to converse could break
|
||||
them. Guards the deliberate scope against an over-broad "any opaque ARN ->
|
||||
converse" generalization.
|
||||
"""
|
||||
for arn_segment in (
|
||||
"provisioned-model/abcdefgh1234",
|
||||
"imported-model/abcdefgh1234",
|
||||
"custom-model-deployment/abcdefgh1234",
|
||||
):
|
||||
route = BedrockModelInfo.get_bedrock_route(
|
||||
model=f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}"
|
||||
)
|
||||
assert route == "invoke", f"{arn_segment} should stay on invoke route"
|
||||
@pytest.mark.parametrize(
|
||||
"arn_segment",
|
||||
["provisioned-model/abcdefgh1234", "imported-model/abcdefgh1234", "custom-model-deployment/abcdefgh1234"],
|
||||
)
|
||||
def test_bare_opaque_arn_with_no_invoke_provider_routes_to_converse(arn_segment: str):
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
|
||||
model: Final = f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}"
|
||||
assert BaseAWSLLM.get_bedrock_invoke_provider(model) is None
|
||||
assert BedrockModelInfo.get_bedrock_route(model) == "converse"
|
||||
assert BedrockModelInfo.get_bedrock_route(f"bedrock/llama/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}") == "invoke"
|
||||
|
||||
|
||||
def test_govcloud_cross_region_inference_prefix():
|
||||
|
|
@ -1098,6 +1091,84 @@ def test_unmapped_openai_family_model_routes_to_converse():
|
|||
assert BedrockModelInfo.get_bedrock_route(imported) == "openai"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"unmapped",
|
||||
[
|
||||
"bedrock/minimax.minimax-m99-unmapped",
|
||||
"bedrock/us.nvidia.nemotron-99b-unmapped",
|
||||
"bedrock/writer.palmyra-x99-unmapped",
|
||||
"bedrock/zai.glm-99-unmapped",
|
||||
"bedrock/google.gemma-99-unmapped",
|
||||
"bedrock/global.xai.grok-99-unmapped",
|
||||
],
|
||||
)
|
||||
def test_unmapped_converse_only_family_model_routes_to_converse(unmapped: str):
|
||||
import litellm
|
||||
|
||||
assert BedrockModelInfo.get_base_model(unmapped) not in litellm.bedrock_converse_models
|
||||
assert BedrockModelInfo.get_bedrock_route(unmapped) == "converse"
|
||||
|
||||
|
||||
def test_unmapped_model_of_family_with_invoke_handler_still_routes_to_invoke():
|
||||
from typing import Final
|
||||
|
||||
import litellm
|
||||
|
||||
unmapped: Final = "bedrock/meta.llama99-unmapped-v1:0"
|
||||
assert BedrockModelInfo.get_base_model(unmapped) not in litellm.bedrock_converse_models
|
||||
assert BedrockModelInfo.get_bedrock_route(unmapped) == "invoke"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "model_path", "response_json"),
|
||||
[
|
||||
(
|
||||
"bedrock/us.newvendor.model-1",
|
||||
b"/model/us.newvendor.model-1/converse",
|
||||
{
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2},
|
||||
},
|
||||
),
|
||||
(
|
||||
"bedrock/mistral.mistral-99-unmapped-v1:0",
|
||||
b"/model/mistral.mistral-99-unmapped-v1:0/invoke",
|
||||
{"outputs": [{"text": "ok", "stop_reason": "stop"}]},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_completion_on_unmapped_model_posts_converse_unless_its_id_names_an_invoke_provider(
|
||||
model: str, model_path: bytes, response_json: dict[str, object]
|
||||
):
|
||||
import httpx
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
assert BedrockModelInfo.get_base_model(model) not in litellm.bedrock_converse_models
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
sent.append(request)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json=response_json,
|
||||
headers={"x-amzn-bedrock-input-token-count": "1", "x-amzn-bedrock-output-token-count": "1"},
|
||||
)
|
||||
|
||||
response: Final = litellm.completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "ok"
|
||||
assert [request.url.raw_path for request in sent] == [model_path]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "expected"),
|
||||
[
|
||||
|
|
|
|||
|
|
@ -24,10 +24,8 @@ class TestGetBedrockRoute:
|
|||
def test_nova2_prefix_routes_to_converse(self):
|
||||
assert BedrockModelInfo.get_bedrock_route(NOVA2_MODEL) == "converse"
|
||||
|
||||
def test_plain_arn_routes_to_invoke(self):
|
||||
# Without spec prefix, ARN doesn't match converse models
|
||||
result = BedrockModelInfo.get_bedrock_route(f"bedrock/{NOVA_ARN}")
|
||||
assert result == "invoke"
|
||||
def test_plain_arn_without_an_invoke_provider_routes_to_converse(self):
|
||||
assert BedrockModelInfo.get_bedrock_route(f"bedrock/{NOVA_ARN}") == "converse"
|
||||
|
||||
|
||||
class TestGetBedrockBaseModel:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue