From dc903ab3bc1a6d97809c92f573f8b589af91ac8e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:50:00 +0000 Subject: [PATCH] fix(bedrock): apply the unknown-model defaults only to bare ids LiteLLM knows nothing about A bare Bedrock id with no cost-map row and no Invoke vendor gets Converse's open params, its reasoning_effort as sent, lookaround-free tool schemas, and a 404 for AWS's invalid-identifier reply, which keeps AWS's response headers. An explicit route like converse/, an ARN, or an id LiteLLM recognizes keeps main's params, regexes and error status, so drop_params still drops tools there --- .../exception_mapping_utils.py | 16 ++- .../bedrock/chat/converse_transformation.py | 22 ++-- litellm/llms/bedrock/common_utils.py | 35 ++++-- .../test_exception_mapping_utils.py | 19 ++- .../chat/test_converse_transformation.py | 109 +++++++++++++++--- 5 files changed, 155 insertions(+), 46 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index ea225d542cd..d4e88ce6309 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import ( ) from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string +from litellm.llms.bedrock.common_utils import bedrock_model_is_unknown from litellm.types.utils import LlmProviders from ..exceptions import ( @@ -901,13 +902,15 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE: Final = ( ) -def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPException) -> httpx.Response: +def _bedrock_response_with_status(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response: original_response: Final = getattr(original_exception, "response", None) if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None: - return httpx.Response(status_code=400, request=httpx.Request(method="POST", url="https://api.openai.com/v1/")) - if original_response.status_code == 400: + return httpx.Response( + status_code=status_code, request=httpx.Request(method="POST", url="https://api.openai.com/v1/") + ) + if original_response.status_code == status_code: return original_response - return httpx.Response(status_code=400, headers=original_response.headers, request=original_response.request) + return httpx.Response(status_code=status_code, headers=original_response.headers, request=original_response.request) def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: @@ -943,7 +946,7 @@ def _map_bedrock_exception( message=mantle_context_window_message, model=model, llm_provider=custom_llm_provider, - response=_bedrock_mantle_context_window_response(original_exception), + response=_bedrock_response_with_status(original_exception, 400), ) if ( "too many tokens" in error_str @@ -966,11 +969,12 @@ def _map_bedrock_exception( llm_provider="bedrock", response=getattr(original_exception, "response", None), ) - elif "The provided model identifier is invalid" in error_str: + elif "The provided model identifier is invalid" in error_str and bedrock_model_is_unknown(model): raise NotFoundError( message=f"BedrockException - {error_str}", model=model, llm_provider="bedrock", + response=_bedrock_response_with_status(original_exception, 404), ) elif "Malformed input request" in error_str: raise BadRequestError( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index ed1c0623e9d..9eb5d3433d3 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -61,6 +61,7 @@ from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_image from litellm.llms.bedrock.common_utils import ( bedrock_chat_uses_native_structured_output, bedrock_cleaned_output_schema, + bedrock_model_is_unknown, bedrock_model_supports_regex_lookaround, bedrock_native_output_format, bedrock_native_output_is_opt_in, @@ -116,14 +117,12 @@ from litellm.utils import ( token_counter, ) -from ..base_aws_llm import BaseAWSLLM from ..common_utils import ( BedrockError, BedrockModelInfo, bedrock_cache_point_ttl_supported, bedrock_capability_model, bedrock_converse_supports_parallel_tool_use_config, - bedrock_cost_map_has_row, bedrock_model_accepts_cache_points, bedrock_reasoning_effort_disabled, bedrock_rejects_stop_sequences, @@ -528,13 +527,14 @@ class AmazonConverseConfig(BaseConfig): """ Handle the reasoning_effort parameter based on the model type. - - GPT-OSS and DeepSeek V3 models: passed through unchanged via additionalModelRequestFields. + - GPT-OSS, DeepSeek V3 and unknown models (``bedrock_model_is_unknown``): passed through unchanged + via additionalModelRequestFields. - OpenAI GPT-5.x and GPT-6 models: mapped to ``reasoning.effort`` via additionalModelRequestFields. - Nova 2 models: transformed to reasoningConfig. - Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on adaptive Claude 4.6 / 4.7). """ - if "gpt-oss" in model or "deepseek" in model: + if "gpt-oss" in model or "deepseek" in model or bedrock_model_is_unknown(model): optional_params["reasoning_effort"] = reasoning_effort elif self._is_openai_gpt_reasoning_model(model): reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort} @@ -662,13 +662,6 @@ class AmazonConverseConfig(BaseConfig): return flag return True - @staticmethod - def _has_no_cost_map_row_or_invoke_provider(model: str, base_model: str) -> bool: - candidates: Final = (model, base_model, *(f"{prefix}{base_model}" for prefix in ("global.", "us.", "eu."))) - if bedrock_cost_map_has_row(candidates): - return False - return BaseAWSLLM.get_bedrock_invoke_provider(model) is None - def get_supported_openai_params(self, model: str) -> list[str]: from litellm.utils import supports_function_calling @@ -687,9 +680,7 @@ class AmazonConverseConfig(BaseConfig): "parallel_tool_calls", ] - base_model: Final = BedrockModelInfo.get_base_model(model) - - if "arn" in model or self._has_no_cost_map_row_or_invoke_provider(model=model, base_model=base_model): + if "arn" in model or bedrock_model_is_unknown(model): supported_params.append("tools") supported_params.append("tool_choice") supported_params.append("thinking") @@ -700,6 +691,9 @@ class AmazonConverseConfig(BaseConfig): supported_params.append("web_search_options") return supported_params + ## Filter out 'cross-region' from model name + base_model: Final = BedrockModelInfo.get_base_model(model) + if ( base_model.startswith("anthropic") or base_model.startswith("mistral") diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 31452e7e4e7..e8bb1cfd4cb 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -10,7 +10,7 @@ import json import os import re import warnings -from collections.abc import Iterable, Iterator, Mapping, Sequence +from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias @@ -1092,10 +1092,6 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None: return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw) -def bedrock_cost_map_has_row(keys: Iterable[str]) -> bool: - return any(_model_cost_entry(key) is not None for key in keys) - - def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]: return tuple( _model_cost_entry(key) @@ -1559,8 +1555,8 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[ The deployment's ``model_info.supports_regex_lookaround`` wins, then the ``model_prices_and_context_window.json`` entry of its ``base_model``, then the - entry of ``model`` itself. A listed model nobody flagged keeps its schema as sent, - and a model the cost map does not list drops the lookarounds. + entry of ``model`` itself. A model nobody flagged keeps its schema as sent, unless + every candidate is a bare id that ``bedrock_model_is_unknown`` describes. """ params: Final = litellm_params or {} deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround") @@ -1572,7 +1568,26 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[ flag: Final = next((flag for flag in flags if flag is not None), None) if flag is not None: return flag - return any(_bedrock_cost_map_lists(candidate) for candidate in candidates) + return not all(bedrock_model_is_unknown(candidate) for candidate in candidates) + + +def bedrock_model_is_unknown(model: str) -> bool: + """ + Whether ``model`` is a bare Bedrock model id that no cost-map row describes and whose vendor + has no Invoke handler, so it routes to Converse with nothing to say what the model accepts. + + An id carrying a route or provider prefix, a region path, or an ARN is never unknown. + """ + model_id: Final = model.removeprefix("bedrock/") + if "/" in model_id: + return False + base: Final = get_bedrock_base_model(model_id) + regional_ids: Final = (f"{region}.{base}" for region in get_bedrock_cross_region_inference_regions()) + if any(_model_cost_entry(key) is not None for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)): + return False + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + return BaseAWSLLM.get_bedrock_invoke_provider(model_id) is None _BedrockConverseModelFlag: TypeAlias = Literal[ @@ -1586,10 +1601,6 @@ def _bedrock_model_info_candidates(model: str) -> tuple[str, ...]: return tuple(dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base)))) -def _bedrock_cost_map_lists(model: str) -> bool: - return bedrock_cost_map_has_row(_bedrock_model_info_candidates(model)) - - def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None: for candidate in _bedrock_model_info_candidates(model): with contextlib.suppress(Exception): diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 55e7c47e2c9..f8e753ae4e7 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -58,7 +58,6 @@ def test_openai_cyber_policy_uses_the_structured_error_code(shape: str, code: st assert caught.value.response.headers["x-request-id"] == "cyber-policy-request" - def _debug_info_for_api_base(configured_api_base: str) -> str: request: Final = httpx.Request("POST", "https://gateway.example/v1/chat/completions") original: Final = OpenAIError( @@ -1468,6 +1467,24 @@ def test_bedrock_mantle_400_maps_to_bad_request(): assert type(excinfo.value) is litellm.BadRequestError +class _ErrorWithoutResponse(Exception): + def __init__(self, message: str, status_code: int) -> None: + super().__init__(message) + self.message = message + self.status_code = status_code + + +def test_bedrock_invalid_model_identifier_without_a_provider_response_still_answers_404(): + with pytest.raises(litellm.NotFoundError) as excinfo: + exception_type( + model="us.newvendor.typo-1", + original_exception=_ErrorWithoutResponse("The provided model identifier is invalid.", 400), + custom_llm_provider="bedrock", + ) + + assert excinfo.value.response.status_code == 404 + + def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded(): from litellm.llms.base_llm.chat.transformation import BaseLLMException diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 285941c97d3..40b0511582a 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -1214,17 +1214,29 @@ def test_transform_request_drops_lookaround_regex_when_the_deployment_model_info assert properties == _LOOKAROUND_FREE_PROPERTIES +def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list(): + assert "us.newvendor.model-1" not in litellm.model_cost + + assert _tool_schema_properties("us.newvendor.model-1", _ARTIFACT_DATA_ANTHROPIC_TOOL) == _LOOKAROUND_FREE_PROPERTIES + + @pytest.mark.parametrize( "model", [ - "us.newvendor.model-1", + "converse/us.newvendor.model-1", "arn:aws:bedrock:us-east-1:123456789012:custom-model-deployment/abc123", + "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", + "us.anthropic.claude-unreleased-v1:0", + "eu.openai.gpt-unreleased", + "apac.newvendor.listed-1", ], ) -def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list(model): - assert model not in litellm.model_cost +def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_litellm_still_recognizes(model, monkeypatch): + monkeypatch.setitem(litellm.model_cost, "us.newvendor.listed-1", {"litellm_provider": "bedrock_converse"}) - assert _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL) == _LOOKAROUND_FREE_PROPERTIES + properties = _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL) + + assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"] def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_whose_deployment_opts_out(): @@ -5408,7 +5420,6 @@ def test_recursive_output_config_format_follows_the_profile_native_key( assert ("outputConfig" in request) is forwarded - def test_sonnet_4_6_json_schema_still_uses_native_output_config(monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) @@ -9243,31 +9254,65 @@ def test_bedrock_tools_transformation_valid_params(): assert "test" in result[0]["toolSpec"]["inputSchema"]["json"]["required"] -def test_not_found_error(): - sent: Final[list[httpx.Request]] = [] - +def _invalid_model_identifier_reply(sent: list[httpx.Request]): def respond(request: httpx.Request) -> httpx.Response: sent.append(request) return httpx.Response( 400, json={"message": "The provided model identifier is invalid."}, - headers={"x-amzn-ErrorType": "ValidationException"}, + headers={"x-amzn-ErrorType": "ValidationException", "x-amzn-RequestId": "req-123"}, ) + return respond + + +def test_not_found_error(): + sent: Final[list[httpx.Request]] = [] + with pytest.raises(litellm.NotFoundError) as exc_info: completion( model="bedrock/bad_model", messages=[{"role": "user", "content": "What is the meaning of life"}], - client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), + client=HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent))) + ), aws_access_key_id="testing", aws_secret_access_key="testing", aws_region_name="us-west-2", ) assert exc_info.value.status_code == 404 + assert exc_info.value.response.status_code == 404 + assert exc_info.value.response.headers["x-amzn-requestid"] == "req-123" assert [request.url.raw_path for request in sent] == [b"/model/bad_model/converse"] +@pytest.mark.parametrize( + "model, path", + [ + ("bedrock/converse/bad_model", b"/model/bad_model/converse"), + ("bedrock/anthropic.claude-typo-v1:0", b"/model/anthropic.claude-typo-v1:0/invoke"), + ], +) +def test_invalid_model_identifier_stays_a_bad_request_for_a_model_litellm_recognizes(model, path): + sent: Final[list[httpx.Request]] = [] + + with pytest.raises(litellm.BadRequestError) as exc_info: + completion( + model=model, + messages=[{"role": "user", "content": "What is the meaning of life"}], + client=HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent))) + ), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert exc_info.value.status_code == 400 + assert [request.url.raw_path for request in sent] == [path] + + _WEATHER_TOOL: Final = { "type": "function", "function": { @@ -9287,7 +9332,9 @@ def _converse_tool_call_reply(sent: list[httpx.Request]): "output": { "message": { "role": "assistant", - "content": [{"toolUse": {"toolUseId": "t1", "name": "get_weather", "input": {"city": "Paris"}}}], + "content": [ + {"toolUse": {"toolUseId": "t1", "name": "get_weather", "input": {"city": "Paris"}}} + ], } }, "stopReason": "tool_use", @@ -9299,7 +9346,7 @@ def _converse_tool_call_reply(sent: list[httpx.Request]): def test_model_missing_from_cost_map_sends_tools_to_converse(): - model: Final = "bedrock/converse/us.newvendor.model-1" + model: Final = "bedrock/us.newvendor.model-1" assert "us.newvendor.model-1" not in litellm.model_cost sent: Final[list[httpx.Request]] = [] @@ -9321,6 +9368,39 @@ def test_model_missing_from_cost_map_sends_tools_to_converse(): assert tool_config["toolChoice"] == {"auto": {}} +def test_explicit_converse_route_to_an_unlisted_model_still_drops_tools_under_drop_params(): + sent: Final[list[httpx.Request]] = [] + + completion( + model="bedrock/converse/us.newvendor.model-1", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + drop_params=True, + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert "toolConfig" not in json.loads(sent[0].content) + + +def test_unknown_model_gets_reasoning_effort_as_sent(): + sent: Final[list[httpx.Request]] = [] + + completion( + model="bedrock/us.newvendor.model-1", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + reasoning_effort="high", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert json.loads(sent[0].content)["additionalModelRequestFields"] == {"reasoning_effort": "high"} + + def test_mapped_model_without_function_calling_still_rejects_tools(): model: Final = "bedrock/converse/meta.llama3-8b-instruct-v1:0" assert "meta.llama3-8b-instruct-v1:0" in litellm.model_cost @@ -10810,7 +10890,10 @@ def test_uses_opt_in_native_output_reads_the_base_model_of_an_application_infere _CALLER_OUTPUT_CONFIG_WITH_A_SCHEMA_OBJECT: Final = { - "textFormat": {"type": "json_schema", "structure": {"jsonSchema": {"name": "capital", "schema": {"type": "object"}}}} + "textFormat": { + "type": "json_schema", + "structure": {"jsonSchema": {"name": "capital", "schema": {"type": "object"}}}, + } } _BEDROCK_SCHEMA_TYPE_ERROR: Final = "outputConfig.textFormat.structure.jsonSchema.schema must be a string"