This commit is contained in:
Siddhant Badola 2026-10-30 01:21:02 +00:00 • committed by GitHub
commit 6cb203dc21
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 444 additions and 108 deletions

View file

@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import (
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
from litellm.llms.bedrock.common_utils import bedrock_model_is_unknown
from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error
from litellm.types.utils import LlmProviders
@ -921,13 +922,15 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE: Final = (
)
def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPException) -> httpx.Response:
def _bedrock_response_with_status(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response:
original_response: Final = getattr(original_exception, "response", None)
if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None:
return httpx.Response(status_code=400, request=httpx.Request(method="POST", url="https://api.openai.com/v1/"))
if original_response.status_code == 400:
return httpx.Response(
status_code=status_code, request=httpx.Request(method="POST", url="https://api.openai.com/v1/")
)
if original_response.status_code == status_code:
return original_response
return httpx.Response(status_code=400, headers=original_response.headers, request=original_response.request)
return httpx.Response(status_code=status_code, headers=original_response.headers, request=original_response.request)
def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
@ -963,7 +966,7 @@ def _map_bedrock_exception(
message=mantle_context_window_message,
model=model,
llm_provider=custom_llm_provider,
response=_bedrock_mantle_context_window_response(original_exception),
response=_bedrock_response_with_status(original_exception, 400),
)
if (
"too many tokens" in error_str
@ -986,6 +989,13 @@ def _map_bedrock_exception(
llm_provider="bedrock",
response=getattr(original_exception, "response", None),
)
elif "The provided model identifier is invalid" in error_str and bedrock_model_is_unknown(model):
raise NotFoundError(
message=f"BedrockException - {error_str}",
model=model,
llm_provider="bedrock",
response=_bedrock_response_with_status(original_exception, 404),
)
elif "Malformed input request" in error_str:
raise BadRequestError(
message=f"BedrockException - {error_str}",

View file

@ -63,6 +63,7 @@ from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_image
from litellm.llms.bedrock.common_utils import (
bedrock_chat_uses_native_structured_output,
bedrock_cleaned_output_schema,
bedrock_model_is_unknown,
bedrock_model_supports_regex_lookaround,
bedrock_native_output_format,
bedrock_native_output_is_opt_in,
@ -529,13 +530,14 @@ class AmazonConverseConfig(BaseConfig):
"""
Handle the reasoning_effort parameter based on the model type.
- GPT-OSS and DeepSeek V3 models: passed through unchanged via additionalModelRequestFields.
- GPT-OSS, DeepSeek V3 and unknown models (``bedrock_model_is_unknown``): passed through unchanged
via additionalModelRequestFields.
- OpenAI GPT-5.x and GPT-6 models: mapped to ``reasoning.effort`` via additionalModelRequestFields.
- Nova 2 models: transformed to reasoningConfig.
- Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on
adaptive Claude 4.6 / 4.7).
"""
if "gpt-oss" in model or "deepseek" in model:
if "gpt-oss" in model or "deepseek" in model or bedrock_model_is_unknown(model):
optional_params["reasoning_effort"] = reasoning_effort
elif self._is_openai_gpt_reasoning_model(model):
reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort}
@ -681,7 +683,7 @@ class AmazonConverseConfig(BaseConfig):
"parallel_tool_calls",
]
if "arn" in model: # we can't infer the model from the arn, so just add all params
if "arn" in model or bedrock_model_is_unknown(model):
supported_params.append("tools")
supported_params.append("tool_choice")
supported_params.append("thinking")

View file

@ -2,7 +2,7 @@ import copy
import json
import time
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, cast, get_args
from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
@ -74,10 +74,6 @@ _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
"{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
)
_INVOKE_PROVIDERS: Final = TypeAdapter(tuple[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, ...]).validate_python(
get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL)
)
def _response_header(headers: Mapping[str, str], name: str) -> str | None:
return headers.get(name, None)
@ -562,62 +558,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
"""
return False
@staticmethod
def get_bedrock_invoke_provider(
model: str,
) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
"""
Helper function to get the bedrock provider from the model
handles 4 scenarios:
1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama`
4. model=us.amazon.nova-pro-v1:0 -> Returns `nova`
"""
if model.startswith("invoke/"):
model = model.replace("invoke/", "", 1)
# Special case: Check for "nova" in model name first (before "amazon")
# This handles amazon.nova-* models which would otherwise match "amazon" (Titan)
if "nova" in model.lower():
if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova")
_split_model: Final = model.split(".")[0]
if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model)
# If not a known provider, check for pattern with two slashes
provider = AmazonInvokeConfig._get_provider_from_model_path(model)
if provider is not None:
return provider
for provider in _INVOKE_PROVIDERS:
if provider in model:
return provider
return None
@staticmethod
def _get_provider_from_model_path(
model_path: str,
) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
"""
Helper function to get the provider from a model path with format: provider/model-name
Args:
model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name')
Returns:
Optional[str]: The provider name, or None if no valid provider found
"""
parts: Final = model_path.split("/")
if len(parts) >= 1:
provider: Final = parts[0]
if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider)
return None
def convert_messages_to_prompt(self, model: str, messages, provider, custom_prompt_dict) -> tuple[str, list | None]:
# handle anthropic prompts and amazon titan prompts
prompt = ""

View file

@ -1098,6 +1098,11 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None:
return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw)
def _cost_map_row_names_a_provider(key: str) -> bool:
entry: Final = _model_cost_entry(key)
return entry is not None and entry.get("litellm_provider") is not None
def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]:
return tuple(
_model_cost_entry(key)
@ -1567,7 +1572,8 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
The deployment's ``model_info.supports_regex_lookaround`` wins, then the
``model_prices_and_context_window.json`` entry of its ``base_model``, then the
entry of ``model`` itself. A model nobody flagged keeps its schema as sent.
entry of ``model`` itself. A model nobody flagged keeps its schema as sent, unless
no candidate names a model LiteLLM knows, whatever route prefix it carries.
"""
params: Final = litellm_params or {}
deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround")
@ -1576,7 +1582,25 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
base_model: Final = params.get("base_model")
candidates: Final = (*((base_model,) if isinstance(base_model, str) else ()), model)
flags: Final = (_bedrock_converse_model_flag(candidate, "supports_regex_lookaround") for candidate in candidates)
return next((flag for flag in flags if flag is not None), True)
flag: Final = next((flag for flag in flags if flag is not None), None)
if flag is not None:
return flag
return not all(bedrock_model_is_unknown(split_bedrock_region_path(candidate)[1]) for candidate in candidates)
def bedrock_model_is_unknown(model: str) -> bool:
routed_id: Final = model.removeprefix("bedrock/")
region, separator, regional_id = routed_id.partition("/")
model_id: Final = regional_id if separator and region in get_all_bedrock_regions() else routed_id
if "/" in model_id:
return False
base: Final = get_bedrock_base_model(model_id)
regional_ids: Final = (f"{region}.{base}" for region in get_bedrock_cross_region_inference_regions())
if any(_cost_map_row_names_a_provider(key) for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)):
return False
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
return BaseAWSLLM.get_bedrock_invoke_provider(model_id) is None
_BedrockConverseModelFlag: TypeAlias = Literal[
@ -1585,10 +1609,13 @@ _BedrockConverseModelFlag: TypeAlias = Literal[
]
def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None:
def _bedrock_model_info_candidates(model: str) -> tuple[str, ...]:
base: Final = get_bedrock_base_model(model)
candidates: Final = dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base)))
for candidate in candidates:
return tuple(dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base))))
def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None:
for candidate in _bedrock_model_info_candidates(model):
with contextlib.suppress(Exception):
model_info = get_cached_model_info()(
model=candidate,
@ -1766,7 +1793,8 @@ class BedrockModelInfo(BaseLLMModelInfo):
default (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in;
``request_params`` (the caller's chat params) sends such a request to Converse when it needs a
feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family
model stays on Converse without the prefix.
model stays on Converse without the prefix. A model the cost map does not list as Converse goes to
Invoke only when its id names an Invoke provider, and to Converse otherwise.
"""
route_mappings: dict[
str,
@ -1822,6 +1850,10 @@ class BedrockModelInfo(BaseLLMModelInfo):
return "converse"
if _OPENAI_FAMILY_MODEL_RE.search(base_model):
return "converse"
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
if BaseAWSLLM.get_bedrock_invoke_provider(model) is None:
return "converse"
return "invoke"
@staticmethod

View file

@ -59,7 +59,6 @@ def test_openai_cyber_policy_uses_the_structured_error_code(shape: str, code: st
assert caught.value.response.headers["x-request-id"] == "cyber-policy-request"
def _debug_info_for_api_base(configured_api_base: str) -> str:
request: Final = httpx.Request("POST", "https://gateway.example/v1/chat/completions")
original: Final = OpenAIError(
@ -1601,6 +1600,24 @@ def test_bedrock_mantle_400_maps_to_bad_request():
assert type(excinfo.value) is litellm.BadRequestError
class _ErrorWithoutResponse(Exception):
def __init__(self, message: str, status_code: int) -> None:
super().__init__(message)
self.message = message
self.status_code = status_code
def test_bedrock_invalid_model_identifier_without_a_provider_response_still_answers_404():
with pytest.raises(litellm.NotFoundError) as excinfo:
exception_type(
model="us.newvendor.typo-1",
original_exception=_ErrorWithoutResponse("The provided model identifier is invalid.", 400),
custom_llm_provider="bedrock",
)
assert excinfo.value.response.status_code == 404
def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded():
from litellm.llms.base_llm.chat.transformation import BaseLLMException

View file

@ -1221,6 +1221,97 @@ def test_transform_request_drops_lookaround_regex_when_the_deployment_model_info
assert properties == _LOOKAROUND_FREE_PROPERTIES
@pytest.mark.parametrize(
("model", "litellm_params"),
[
("us.newvendor.model-1", None),
("converse/us.newvendor.model-1", None),
("us-east-1/us.newvendor.model-1", None),
("us.newvendor.model-1", {"base_model": "bedrock/converse/us.newvendor.model-1"}),
],
)
def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list(model, litellm_params):
assert "us.newvendor.model-1" not in litellm.model_cost
properties: Final = _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL, litellm_params)
assert properties == _LOOKAROUND_FREE_PROPERTIES
@pytest.mark.parametrize(
"model",
[
"arn:aws:bedrock:us-east-1:123456789012:custom-model-deployment/abc123",
"arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123",
"us.anthropic.claude-unreleased-v1:0",
"eu.openai.gpt-unreleased",
"apac.newvendor.listed-1",
],
)
def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_litellm_still_recognizes(model, monkeypatch):
monkeypatch.setitem(litellm.model_cost, "us.newvendor.listed-1", {"litellm_provider": "bedrock_converse"})
properties = _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL)
assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"]
def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_whose_deployment_opts_out():
properties = _tool_schema_properties(
"us.newvendor.model-1", _ARTIFACT_DATA_ANTHROPIC_TOOL, {"model_info": {"supports_regex_lookaround": True}}
)
assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"]
@pytest.mark.parametrize(
"deployment_params",
[
{"model": "bedrock/us.newvendor.model-1"},
{"model": "bedrock/converse/us.newvendor.model-1", "allowed_openai_params": ["tools"]},
],
)
def test_a_router_deployment_of_a_model_the_cost_map_does_not_list_drops_lookaround_regex(
respx_mock, deployment_params
):
route: Final = respx_mock.post(
"https://bedrock-runtime.us-east-1.amazonaws.com/model/us.newvendor.model-1/converse"
).mock(
return_value=httpx.Response(
200,
json={
"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
"stopReason": "end_turn",
"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2},
},
)
)
router: Final = Router(
model_list=[
{
"model_name": "unlisted",
"litellm_params": {
**deployment_params,
"aws_access_key_id": "fake-access-key",
"aws_secret_access_key": "fake-secret-key",
"aws_region_name": "us-east-1",
},
"model_info": {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6},
}
]
)
router.completion(
model="unlisted",
messages=[{"role": "user", "content": "hi"}],
tools=[_ARTIFACT_DATA_OPENAI_TOOL],
client=HTTPHandler(),
)
tool: Final = json.loads(route.calls[0].request.content)["toolConfig"]["tools"][0]
assert tool["toolSpec"]["inputSchema"]["json"]["properties"] == _LOOKAROUND_FREE_PROPERTIES
def test_transform_request_keeps_lookaround_regex_when_the_deployment_model_info_opts_out():
properties = _tool_schema_properties(
"global.moonshotai.kimi-k3", _ARTIFACT_DATA_ANTHROPIC_TOOL, {"model_info": {"supports_regex_lookaround": True}}
@ -9189,18 +9280,193 @@ def test_bedrock_tools_transformation_valid_params():
assert "test" in result[0]["toolSpec"]["inputSchema"]["json"]["required"]
def test_not_found_error():
with pytest.raises(litellm.NotFoundError):
completion(
model="bedrock/bad_model",
messages=[
{
"role": "user",
"content": "What is the meaning of life",
}
],
def _invalid_model_identifier_reply(sent: list[httpx.Request]):
def respond(request: httpx.Request) -> httpx.Response:
sent.append(request)
return httpx.Response(
400,
json={"message": "The provided model identifier is invalid."},
headers={"x-amzn-ErrorType": "ValidationException", "x-amzn-RequestId": "req-123"},
)
return respond
@pytest.mark.parametrize("model", ["bedrock/bad_model", "bedrock/us-east-1/bad_model"])
def test_not_found_error(model):
sent: Final[list[httpx.Request]] = []
with pytest.raises(litellm.NotFoundError) as exc_info:
completion(
model=model,
messages=[{"role": "user", "content": "What is the meaning of life"}],
client=HTTPHandler(
client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent)))
),
aws_access_key_id="testing",
aws_secret_access_key="testing",
aws_region_name="us-west-2",
)
assert exc_info.value.status_code == 404
assert exc_info.value.response.status_code == 404
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-123"
assert [request.url.raw_path for request in sent] == [b"/model/bad_model/converse"]
@pytest.mark.parametrize(
"model, path",
[
("bedrock/converse/bad_model", b"/model/bad_model/converse"),
("bedrock/anthropic.claude-typo-v1:0", b"/model/anthropic.claude-typo-v1:0/invoke"),
],
)
def test_invalid_model_identifier_stays_a_bad_request_for_a_model_litellm_recognizes(model, path):
sent: Final[list[httpx.Request]] = []
with pytest.raises(litellm.BadRequestError) as exc_info:
completion(
model=model,
messages=[{"role": "user", "content": "What is the meaning of life"}],
client=HTTPHandler(
client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent)))
),
aws_access_key_id="testing",
aws_secret_access_key="testing",
aws_region_name="us-west-2",
)
assert exc_info.value.status_code == 400
assert [request.url.raw_path for request in sent] == [path]
_WEATHER_TOOL: Final = {
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the weather for a city",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
},
}
def _converse_tool_call_reply(sent: list[httpx.Request]):
def respond(request: httpx.Request) -> httpx.Response:
sent.append(request)
return httpx.Response(
200,
json={
"output": {
"message": {
"role": "assistant",
"content": [
{"toolUse": {"toolUseId": "t1", "name": "get_weather", "input": {"city": "Paris"}}}
],
}
},
"stopReason": "tool_use",
"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2},
},
)
return respond
@pytest.mark.parametrize("model", ["bedrock/us.newvendor.model-1", "bedrock/us-east-1/us.newvendor.model-1"])
def test_model_missing_from_cost_map_sends_tools_to_converse(model):
assert "us.newvendor.model-1" not in litellm.model_cost
sent: Final[list[httpx.Request]] = []
response: Final = completion(
model=model,
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
tools=[_WEATHER_TOOL],
tool_choice="auto",
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
aws_access_key_id="testing",
aws_secret_access_key="testing",
aws_region_name="us-west-2",
)
assert response.choices[0].message.tool_calls[0].function.name == "get_weather"
assert [request.url.raw_path for request in sent] == [b"/model/us.newvendor.model-1/converse"]
tool_config: Final = json.loads(sent[0].content)["toolConfig"]
assert [tool["toolSpec"]["name"] for tool in tool_config["tools"]] == ["get_weather"]
assert tool_config["toolChoice"] == {"auto": {}}
def test_a_price_only_row_for_an_unlisted_model_still_sends_tools_to_converse(monkeypatch):
monkeypatch.setitem(
litellm.model_cost, "us.newvendor.priced-1", {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}
)
sent: Final[list[httpx.Request]] = []
completion(
model="bedrock/us.newvendor.priced-1",
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
tools=[_WEATHER_TOOL],
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
aws_access_key_id="testing",
aws_secret_access_key="testing",
aws_region_name="us-west-2",
)
tool_config: Final = json.loads(sent[0].content)["toolConfig"]
assert [tool["toolSpec"]["name"] for tool in tool_config["tools"]] == ["get_weather"]
def test_explicit_converse_route_to_an_unlisted_model_still_drops_tools_under_drop_params():
sent: Final[list[httpx.Request]] = []
completion(
model="bedrock/converse/us.newvendor.model-1",
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
tools=[_WEATHER_TOOL],
drop_params=True,
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
aws_access_key_id="testing",
aws_secret_access_key="testing",
aws_region_name="us-west-2",
)
assert "toolConfig" not in json.loads(sent[0].content)
def test_unknown_model_gets_reasoning_effort_as_sent():
sent: Final[list[httpx.Request]] = []
completion(
model="bedrock/us.newvendor.model-1",
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
reasoning_effort="high",
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
aws_access_key_id="testing",
aws_secret_access_key="testing",
aws_region_name="us-west-2",
)
assert json.loads(sent[0].content)["additionalModelRequestFields"] == {"reasoning_effort": "high"}
def test_mapped_model_without_function_calling_still_rejects_tools():
model: Final = "bedrock/converse/meta.llama3-8b-instruct-v1:0"
assert "meta.llama3-8b-instruct-v1:0" in litellm.model_cost
assert not litellm.supports_function_calling(model=model)
sent: Final[list[httpx.Request]] = []
with pytest.raises(litellm.UnsupportedParamsError):
completion(
model=model,
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
tools=[_WEATHER_TOOL],
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
aws_access_key_id="testing",
aws_secret_access_key="testing",
aws_region_name="us-west-2",
)
assert sent == []
@pytest.mark.parametrize(
"model, expected_base_model",

View file

@ -214,24 +214,17 @@ def test_system_defined_inference_profile_arn_still_routes_to_converse():
assert route == "converse"
def test_other_opaque_arn_types_still_route_to_invoke():
"""
Only application-inference-profile ARNs are auto-routed to converse. Other
opaque ARNs (provisioned-model, imported-model, custom-model-deployment)
also yield no invoke provider, but they are frequently invoke-only with
provider-specific body formats, so routing them to converse could break
them. Guards the deliberate scope against an over-broad "any opaque ARN ->
converse" generalization.
"""
for arn_segment in (
"provisioned-model/abcdefgh1234",
"imported-model/abcdefgh1234",
"custom-model-deployment/abcdefgh1234",
):
route = BedrockModelInfo.get_bedrock_route(
model=f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}"
)
assert route == "invoke", f"{arn_segment} should stay on invoke route"
@pytest.mark.parametrize(
"arn_segment",
["provisioned-model/abcdefgh1234", "imported-model/abcdefgh1234", "custom-model-deployment/abcdefgh1234"],
)
def test_bare_opaque_arn_with_no_invoke_provider_routes_to_converse(arn_segment: str):
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
model: Final = f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}"
assert BaseAWSLLM.get_bedrock_invoke_provider(model) is None
assert BedrockModelInfo.get_bedrock_route(model) == "converse"
assert BedrockModelInfo.get_bedrock_route(f"bedrock/llama/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}") == "invoke"
def test_govcloud_cross_region_inference_prefix():
@ -1098,6 +1091,84 @@ def test_unmapped_openai_family_model_routes_to_converse():
assert BedrockModelInfo.get_bedrock_route(imported) == "openai"
@pytest.mark.parametrize(
"unmapped",
[
"bedrock/minimax.minimax-m99-unmapped",
"bedrock/us.nvidia.nemotron-99b-unmapped",
"bedrock/writer.palmyra-x99-unmapped",
"bedrock/zai.glm-99-unmapped",
"bedrock/google.gemma-99-unmapped",
"bedrock/global.xai.grok-99-unmapped",
],
)
def test_unmapped_converse_only_family_model_routes_to_converse(unmapped: str):
import litellm
assert BedrockModelInfo.get_base_model(unmapped) not in litellm.bedrock_converse_models
assert BedrockModelInfo.get_bedrock_route(unmapped) == "converse"
def test_unmapped_model_of_family_with_invoke_handler_still_routes_to_invoke():
from typing import Final
import litellm
unmapped: Final = "bedrock/meta.llama99-unmapped-v1:0"
assert BedrockModelInfo.get_base_model(unmapped) not in litellm.bedrock_converse_models
assert BedrockModelInfo.get_bedrock_route(unmapped) == "invoke"
@pytest.mark.parametrize(
("model", "model_path", "response_json"),
[
(
"bedrock/us.newvendor.model-1",
b"/model/us.newvendor.model-1/converse",
{
"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
"stopReason": "end_turn",
"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2},
},
),
(
"bedrock/mistral.mistral-99-unmapped-v1:0",
b"/model/mistral.mistral-99-unmapped-v1:0/invoke",
{"outputs": [{"text": "ok", "stop_reason": "stop"}]},
),
],
)
def test_completion_on_unmapped_model_posts_converse_unless_its_id_names_an_invoke_provider(
model: str, model_path: bytes, response_json: dict[str, object]
):
import httpx
from litellm.llms.custom_httpx.http_handler import HTTPHandler
assert BedrockModelInfo.get_base_model(model) not in litellm.bedrock_converse_models
sent: Final[list[httpx.Request]] = []
def respond(request: httpx.Request) -> httpx.Response:
sent.append(request)
return httpx.Response(
200,
json=response_json,
headers={"x-amzn-bedrock-input-token-count": "1", "x-amzn-bedrock-output-token-count": "1"},
)
response: Final = litellm.completion(
model=model,
messages=[{"role": "user", "content": "hello"}],
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))),
aws_access_key_id="testing",
aws_secret_access_key="testing",
aws_region_name="us-west-2",
)
assert response.choices[0].message.content == "ok"
assert [request.url.raw_path for request in sent] == [model_path]
@pytest.mark.parametrize(
("model", "expected"),
[

View file

@ -24,10 +24,8 @@ class TestGetBedrockRoute:
def test_nova2_prefix_routes_to_converse(self):
assert BedrockModelInfo.get_bedrock_route(NOVA2_MODEL) == "converse"
def test_plain_arn_routes_to_invoke(self):
# Without spec prefix, ARN doesn't match converse models
result = BedrockModelInfo.get_bedrock_route(f"bedrock/{NOVA_ARN}")
assert result == "invoke"
def test_plain_arn_without_an_invoke_provider_routes_to_converse(self):
assert BedrockModelInfo.get_bedrock_route(f"bedrock/{NOVA_ARN}") == "converse"
class TestGetBedrockBaseModel: