fix(bedrock): apply the unknown-model defaults only to bare ids LiteLLM knows nothing about

A bare Bedrock id with no cost-map row and no Invoke vendor gets Converse's open params, its reasoning_effort as sent, lookaround-free tool schemas, and a 404 for AWS's invalid-identifier reply, which keeps AWS's response headers. An explicit route like converse/, an ARN, or an id LiteLLM recognizes keeps main's params, regexes and error status, so drop_params still drops tools there
This commit is contained in:
mateo-berri 2026-10-29 18:50:00 +00:00
parent f09bf3ac35
commit dc903ab3bc
5 changed files with 155 additions and 46 deletions

View file

@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import (
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
from litellm.llms.bedrock.common_utils import bedrock_model_is_unknown
from litellm.types.utils import LlmProviders
from ..exceptions import (
@ -901,13 +902,15 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE: Final = (
)
def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPException) -> httpx.Response:
def _bedrock_response_with_status(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response:
original_response: Final = getattr(original_exception, "response", None)
if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None:
return httpx.Response(status_code=400, request=httpx.Request(method="POST", url="https://api.openai.com/v1/"))
if original_response.status_code == 400:
return httpx.Response(
status_code=status_code, request=httpx.Request(method="POST", url="https://api.openai.com/v1/")
)
if original_response.status_code == status_code:
return original_response
return httpx.Response(status_code=400, headers=original_response.headers, request=original_response.request)
return httpx.Response(status_code=status_code, headers=original_response.headers, request=original_response.request)
def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
@ -943,7 +946,7 @@ def _map_bedrock_exception(
message=mantle_context_window_message,
model=model,
llm_provider=custom_llm_provider,
response=_bedrock_mantle_context_window_response(original_exception),
response=_bedrock_response_with_status(original_exception, 400),
)
if (
"too many tokens" in error_str
@ -966,11 +969,12 @@ def _map_bedrock_exception(
llm_provider="bedrock",
response=getattr(original_exception, "response", None),
)
elif "The provided model identifier is invalid" in error_str:
elif "The provided model identifier is invalid" in error_str and bedrock_model_is_unknown(model):
raise NotFoundError(
message=f"BedrockException - {error_str}",
model=model,
llm_provider="bedrock",
response=_bedrock_response_with_status(original_exception, 404),
)
elif "Malformed input request" in error_str:
raise BadRequestError(

View file

@ -61,6 +61,7 @@ from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_image
from litellm.llms.bedrock.common_utils import (
bedrock_chat_uses_native_structured_output,
bedrock_cleaned_output_schema,
bedrock_model_is_unknown,
bedrock_model_supports_regex_lookaround,
bedrock_native_output_format,
bedrock_native_output_is_opt_in,
@ -116,14 +117,12 @@ from litellm.utils import (
token_counter,
)
from ..base_aws_llm import BaseAWSLLM
from ..common_utils import (
BedrockError,
BedrockModelInfo,
bedrock_cache_point_ttl_supported,
bedrock_capability_model,
bedrock_converse_supports_parallel_tool_use_config,
bedrock_cost_map_has_row,
bedrock_model_accepts_cache_points,
bedrock_reasoning_effort_disabled,
bedrock_rejects_stop_sequences,
@ -528,13 +527,14 @@ class AmazonConverseConfig(BaseConfig):
"""
Handle the reasoning_effort parameter based on the model type.
- GPT-OSS and DeepSeek V3 models: passed through unchanged via additionalModelRequestFields.
- GPT-OSS, DeepSeek V3 and unknown models (``bedrock_model_is_unknown``): passed through unchanged
via additionalModelRequestFields.
- OpenAI GPT-5.x and GPT-6 models: mapped to ``reasoning.effort`` via additionalModelRequestFields.
- Nova 2 models: transformed to reasoningConfig.
- Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on
adaptive Claude 4.6 / 4.7).
"""
if "gpt-oss" in model or "deepseek" in model:
if "gpt-oss" in model or "deepseek" in model or bedrock_model_is_unknown(model):
optional_params["reasoning_effort"] = reasoning_effort
elif self._is_openai_gpt_reasoning_model(model):
reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort}
@ -662,13 +662,6 @@ class AmazonConverseConfig(BaseConfig):
return flag
return True
@staticmethod
def _has_no_cost_map_row_or_invoke_provider(model: str, base_model: str) -> bool:
candidates: Final = (model, base_model, *(f"{prefix}{base_model}" for prefix in ("global.", "us.", "eu.")))
if bedrock_cost_map_has_row(candidates):
return False
return BaseAWSLLM.get_bedrock_invoke_provider(model) is None
def get_supported_openai_params(self, model: str) -> list[str]:
from litellm.utils import supports_function_calling
@ -687,9 +680,7 @@ class AmazonConverseConfig(BaseConfig):
"parallel_tool_calls",
]
base_model: Final = BedrockModelInfo.get_base_model(model)
if "arn" in model or self._has_no_cost_map_row_or_invoke_provider(model=model, base_model=base_model):
if "arn" in model or bedrock_model_is_unknown(model):
supported_params.append("tools")
supported_params.append("tool_choice")
supported_params.append("thinking")
@ -700,6 +691,9 @@ class AmazonConverseConfig(BaseConfig):
supported_params.append("web_search_options")
return supported_params
## Filter out 'cross-region' from model name
base_model: Final = BedrockModelInfo.get_base_model(model)
if (
base_model.startswith("anthropic")
or base_model.startswith("mistral")

View file

@ -10,7 +10,7 @@ import json
import os
import re
import warnings
from collections.abc import Iterable, Iterator, Mapping, Sequence
from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias
@ -1092,10 +1092,6 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None:
return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw)
def bedrock_cost_map_has_row(keys: Iterable[str]) -> bool:
return any(_model_cost_entry(key) is not None for key in keys)
def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]:
return tuple(
_model_cost_entry(key)
@ -1559,8 +1555,8 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
The deployment's ``model_info.supports_regex_lookaround`` wins, then the
``model_prices_and_context_window.json`` entry of its ``base_model``, then the
entry of ``model`` itself. A listed model nobody flagged keeps its schema as sent,
and a model the cost map does not list drops the lookarounds.
entry of ``model`` itself. A model nobody flagged keeps its schema as sent, unless
every candidate is a bare id that ``bedrock_model_is_unknown`` describes.
"""
params: Final = litellm_params or {}
deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround")
@ -1572,7 +1568,26 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
flag: Final = next((flag for flag in flags if flag is not None), None)
if flag is not None:
return flag
return any(_bedrock_cost_map_lists(candidate) for candidate in candidates)
return not all(bedrock_model_is_unknown(candidate) for candidate in candidates)
def bedrock_model_is_unknown(model: str) -> bool:
"""
Whether ``model`` is a bare Bedrock model id that no cost-map row describes and whose vendor
has no Invoke handler, so it routes to Converse with nothing to say what the model accepts.
An id carrying a route or provider prefix, a region path, or an ARN is never unknown.
"""
model_id: Final = model.removeprefix("bedrock/")
if "/" in model_id:
return False
base: Final = get_bedrock_base_model(model_id)
regional_ids: Final = (f"{region}.{base}" for region in get_bedrock_cross_region_inference_regions())
if any(_model_cost_entry(key) is not None for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)):
return False
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
return BaseAWSLLM.get_bedrock_invoke_provider(model_id) is None
_BedrockConverseModelFlag: TypeAlias = Literal[
@ -1586,10 +1601,6 @@ def _bedrock_model_info_candidates(model: str) -> tuple[str, ...]:
return tuple(dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base))))
def _bedrock_cost_map_lists(model: str) -> bool:
return bedrock_cost_map_has_row(_bedrock_model_info_candidates(model))
def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None:
for candidate in _bedrock_model_info_candidates(model):
with contextlib.suppress(Exception):

View file

@ -58,7 +58,6 @@ def test_openai_cyber_policy_uses_the_structured_error_code(shape: str, code: st
assert caught.value.response.headers["x-request-id"] == "cyber-policy-request"
def _debug_info_for_api_base(configured_api_base: str) -> str:
request: Final = httpx.Request("POST", "https://gateway.example/v1/chat/completions")
original: Final = OpenAIError(
@ -1468,6 +1467,24 @@ def test_bedrock_mantle_400_maps_to_bad_request():
assert type(excinfo.value) is litellm.BadRequestError
class _ErrorWithoutResponse(Exception):
def __init__(self, message: str, status_code: int) -> None:
super().__init__(message)
self.message = message
self.status_code = status_code
def test_bedrock_invalid_model_identifier_without_a_provider_response_still_answers_404():
with pytest.raises(litellm.NotFoundError) as excinfo:
exception_type(
model="us.newvendor.typo-1",
original_exception=_ErrorWithoutResponse("The provided model identifier is invalid.", 400),
custom_llm_provider="bedrock",
)
assert excinfo.value.response.status_code == 404
def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded():
from litellm.llms.base_llm.chat.transformation import BaseLLMException

View file

@ -1214,17 +1214,29 @@ def test_transform_request_drops_lookaround_regex_when_the_deployment_model_info
assert properties == _LOOKAROUND_FREE_PROPERTIES
def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list():
assert "us.newvendor.model-1" not in litellm.model_cost
assert _tool_schema_properties("us.newvendor.model-1", _ARTIFACT_DATA_ANTHROPIC_TOOL) == _LOOKAROUND_FREE_PROPERTIES
@pytest.mark.parametrize(
"model",
[
"us.newvendor.model-1",
"converse/us.newvendor.model-1",
"arn:aws:bedrock:us-east-1:123456789012:custom-model-deployment/abc123",
"arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123",
"us.anthropic.claude-unreleased-v1:0",
"eu.openai.gpt-unreleased",
"apac.newvendor.listed-1",
],
)
def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list(model):
assert model not in litellm.model_cost
def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_litellm_still_recognizes(model, monkeypatch):
monkeypatch.setitem(litellm.model_cost, "us.newvendor.listed-1", {"litellm_provider": "bedrock_converse"})
assert _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL) == _LOOKAROUND_FREE_PROPERTIES
properties = _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL)
assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"]
def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_whose_deployment_opts_out():
@ -5408,7 +5420,6 @@ def test_recursive_output_config_format_follows_the_profile_native_key(
assert ("outputConfig" in request) is forwarded
def test_sonnet_4_6_json_schema_still_uses_native_output_config(monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
@ -9243,31 +9254,65 @@ def test_bedrock_tools_transformation_valid_params():
assert "test" in result[0]["toolSpec"]["inputSchema"]["json"]["required"]
def test_not_found_error():
sent: Final[list[httpx.Request]] = []
def _invalid_model_identifier_reply(sent: list[httpx.Request]):
def respond(request: httpx.Request) -> httpx.Response:
sent.append(request)
return httpx.Response(
400,
json={"message": "The provided model identifier is invalid."},
headers={"x-amzn-ErrorType": "ValidationException"},
headers={"x-amzn-ErrorType": "ValidationException", "x-amzn-RequestId": "req-123"},
)
return respond
def test_not_found_error():
sent: Final[list[httpx.Request]] = []
with pytest.raises(litellm.NotFoundError) as exc_info:
completion(
model="bedrock/bad_model",
messages=[{"role": "user", "content": "What is the meaning of life"}],
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))),
client=HTTPHandler(
client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent)))
),
aws_access_key_id="testing",
aws_secret_access_key="testing",
aws_region_name="us-west-2",
)
assert exc_info.value.status_code == 404
assert exc_info.value.response.status_code == 404
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-123"
assert [request.url.raw_path for request in sent] == [b"/model/bad_model/converse"]
@pytest.mark.parametrize(
"model, path",
[
("bedrock/converse/bad_model", b"/model/bad_model/converse"),
("bedrock/anthropic.claude-typo-v1:0", b"/model/anthropic.claude-typo-v1:0/invoke"),
],
)
def test_invalid_model_identifier_stays_a_bad_request_for_a_model_litellm_recognizes(model, path):
sent: Final[list[httpx.Request]] = []
with pytest.raises(litellm.BadRequestError) as exc_info:
completion(
model=model,
messages=[{"role": "user", "content": "What is the meaning of life"}],
client=HTTPHandler(
client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent)))
),
aws_access_key_id="testing",
aws_secret_access_key="testing",
aws_region_name="us-west-2",
)
assert exc_info.value.status_code == 400
assert [request.url.raw_path for request in sent] == [path]
_WEATHER_TOOL: Final = {
"type": "function",
"function": {
@ -9287,7 +9332,9 @@ def _converse_tool_call_reply(sent: list[httpx.Request]):
"output": {
"message": {
"role": "assistant",
"content": [{"toolUse": {"toolUseId": "t1", "name": "get_weather", "input": {"city": "Paris"}}}],
"content": [
{"toolUse": {"toolUseId": "t1", "name": "get_weather", "input": {"city": "Paris"}}}
],
}
},
"stopReason": "tool_use",
@ -9299,7 +9346,7 @@ def _converse_tool_call_reply(sent: list[httpx.Request]):
def test_model_missing_from_cost_map_sends_tools_to_converse():
model: Final = "bedrock/converse/us.newvendor.model-1"
model: Final = "bedrock/us.newvendor.model-1"
assert "us.newvendor.model-1" not in litellm.model_cost
sent: Final[list[httpx.Request]] = []
@ -9321,6 +9368,39 @@ def test_model_missing_from_cost_map_sends_tools_to_converse():
assert tool_config["toolChoice"] == {"auto": {}}
def test_explicit_converse_route_to_an_unlisted_model_still_drops_tools_under_drop_params():
sent: Final[list[httpx.Request]] = []
completion(
model="bedrock/converse/us.newvendor.model-1",
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
tools=[_WEATHER_TOOL],
drop_params=True,
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
aws_access_key_id="testing",
aws_secret_access_key="testing",
aws_region_name="us-west-2",
)
assert "toolConfig" not in json.loads(sent[0].content)
def test_unknown_model_gets_reasoning_effort_as_sent():
sent: Final[list[httpx.Request]] = []
completion(
model="bedrock/us.newvendor.model-1",
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
reasoning_effort="high",
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
aws_access_key_id="testing",
aws_secret_access_key="testing",
aws_region_name="us-west-2",
)
assert json.loads(sent[0].content)["additionalModelRequestFields"] == {"reasoning_effort": "high"}
def test_mapped_model_without_function_calling_still_rejects_tools():
model: Final = "bedrock/converse/meta.llama3-8b-instruct-v1:0"
assert "meta.llama3-8b-instruct-v1:0" in litellm.model_cost
@ -10810,7 +10890,10 @@ def test_uses_opt_in_native_output_reads_the_base_model_of_an_application_infere
_CALLER_OUTPUT_CONFIG_WITH_A_SCHEMA_OBJECT: Final = {
"textFormat": {"type": "json_schema", "structure": {"jsonSchema": {"name": "capital", "schema": {"type": "object"}}}}
"textFormat": {
"type": "json_schema",
"structure": {"jsonSchema": {"name": "capital", "schema": {"type": "object"}}},
}
}
_BEDROCK_SCHEMA_TYPE_ERROR: Final = "outputConfig.textFormat.structure.jsonSchema.schema must be a string"