From ce65873c89681cb2e64465b8fee259a65c9248f7 Mon Sep 17 00:00:00 2001 From: Siddhant Badola Date: Sat, 26 Sep 2026 01:43:22 -0400 Subject: [PATCH 1/9] fix(bedrock): route unmapped converse-only family model ids to converse #42713 stopped Bedrock-native OpenAI ids that are not in the cost map yet from falling to the invoke route. The same gap exists for the other families that have no invoke transformation at all: minimax, nvidia, writer, zai, google and xai. An id from one of them that is newer than the cost map hits get_bedrock_route -> "invoke" -> get_bedrock_invoke_provider() -> None -> BedrockError, when Converse would have served it Add a second family regex for those six and default them to converse, the same way the openai family is handled. _OPENAI_FAMILY_MODEL_RE is left alone because the prompt-caching rule is openai-specific Every shipped id of these families already resolves to a converse entry via its base model (checked against the cost map), so this only changes behaviour for ids that are not in the map yet Tests: six unmapped ids route to converse, and an unmapped meta.* id still routes to invoke since meta has an invoke handler --- litellm/llms/bedrock/common_utils.py | 4 +++ .../llms/bedrock/test_bedrock_common_utils.py | 36 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 5f044897b2c..eff8395a0fe 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -37,6 +37,8 @@ if TYPE_CHECKING: _ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs" _OPENAI_FAMILY_MODEL_RE: Final = re.compile(r"(^|[./])openai\.") +# Families with no invoke transformation; unmapped ids default to converse. +_CONVERSE_ONLY_FAMILY_MODEL_RE: Final = re.compile(r"(^|[./])(minimax|nvidia|writer|zai|google|xai)\.") def error_response_text(response: httpx.Response) -> str: @@ -1220,6 +1222,8 @@ class BedrockModelInfo(BaseLLMModelInfo): return "converse" if _OPENAI_FAMILY_MODEL_RE.search(base_model): return "converse" + if _CONVERSE_ONLY_FAMILY_MODEL_RE.search(base_model): + return "converse" return "invoke" @staticmethod diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index e5118f90e44..4c6b85871ba 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -981,3 +981,39 @@ def test_unmapped_openai_family_model_routes_to_converse(): assert BedrockModelInfo.get_bedrock_route(unmapped) == "converse" imported: Final = "bedrock/openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123" assert BedrockModelInfo.get_bedrock_route(imported) == "openai" + + +@pytest.mark.parametrize( + "unmapped", + [ + "bedrock/minimax.minimax-m99-unmapped", + "bedrock/us.nvidia.nemotron-99b-unmapped", + "bedrock/writer.palmyra-x99-unmapped", + "bedrock/zai.glm-99-unmapped", + "bedrock/google.gemma-99-unmapped", + "bedrock/global.xai.grok-99-unmapped", + ], +) +def test_unmapped_converse_only_family_model_routes_to_converse(unmapped: str): + """Same rule as the OpenAI family, for the other Bedrock-native families with no invoke handler. + + minimax, nvidia, writer, zai, google and xai have no transformation under + ``litellm/llms/bedrock/chat/invoke_transformations``; every id of theirs is served by Converse. + An id newer than the cost map used to fall to the invoke route, where + ``get_bedrock_invoke_provider`` returns ``None`` and the request fails, instead of Converse. + """ + import litellm + + assert BedrockModelInfo.get_base_model(unmapped) not in litellm.bedrock_converse_models + assert BedrockModelInfo.get_bedrock_route(unmapped) == "converse" + + +def test_unmapped_model_of_family_with_invoke_handler_still_routes_to_invoke(): + """The converse-only default is scoped to families without an invoke handler; ``meta`` has one.""" + from typing import Final + + import litellm + + unmapped: Final = "bedrock/meta.llama99-unmapped-v1:0" + assert BedrockModelInfo.get_base_model(unmapped) not in litellm.bedrock_converse_models + assert BedrockModelInfo.get_bedrock_route(unmapped) == "invoke" From f886469d679759629b719e23f5a9926b697137df Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:30:28 -0700 Subject: [PATCH 2/9] fix(bedrock): route a model id with no Invoke provider to Converse A bedrock/ id missing from the cost map fell to the Invoke route, where an id that names no Invoke provider (a new vendor's model, a bare provisioned-model, imported-model, or custom-model-deployment ARN) fails locally with "Unknown provider=None" before any request is sent. Such ids now go to Converse, which replaces the per-family regex with the general rule. Ids that name an Invoke provider and explicit route prefixes are unchanged. --- litellm/llms/bedrock/common_utils.py | 9 +- .../llms/bedrock/test_bedrock_common_utils.py | 92 ++++++++++++++----- 2 files changed, 76 insertions(+), 25 deletions(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 008dad7b385..eaa5ff21c9a 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -50,8 +50,6 @@ BEDROCK_INVOKE_TRACE_FIELD: Final = "amazon-bedrock-trace" _ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs" _OPENAI_FAMILY_MODEL_RE: Final = re.compile(r"(^|[./])openai\.") -# Families with no invoke transformation; unmapped ids default to converse. -_CONVERSE_ONLY_FAMILY_MODEL_RE: Final = re.compile(r"(^|[./])(minimax|nvidia|writer|zai|google|xai)\.") _OPENAI_GPT_VERSION_RE: Final = re.compile(r"(^|[./])openai\.gpt-(\d{1,3})(?!\d)(?:\.(\d{1,3})(?!\d))?") _BEDROCK_RUNTIME_CHAT_COMPLETIONS_DEFAULT_SINCE: Final = (5, 6) _XAI_GROK_MODEL_RE: Final = re.compile(r"(^|[./])xai\.grok-") @@ -1756,7 +1754,8 @@ class BedrockModelInfo(BaseLLMModelInfo): default (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in; ``request_params`` (the caller's chat params) sends such a request to Converse when it needs a feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family - model stays on Converse without the prefix. + model stays on Converse without the prefix. A model the cost map does not list as Converse goes to + Invoke only when its id names an Invoke provider, and to Converse otherwise. """ route_mappings: dict[ str, @@ -1812,7 +1811,9 @@ class BedrockModelInfo(BaseLLMModelInfo): return "converse" if _OPENAI_FAMILY_MODEL_RE.search(base_model): return "converse" - if _CONVERSE_ONLY_FAMILY_MODEL_RE.search(base_model): + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + if BaseAWSLLM.get_bedrock_invoke_provider(model) is None: return "converse" return "invoke" diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index c1bee99160e..f98de8d1c1e 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -214,24 +214,22 @@ def test_system_defined_inference_profile_arn_still_routes_to_converse(): assert route == "converse" -def test_other_opaque_arn_types_still_route_to_invoke(): +@pytest.mark.parametrize( + "arn_segment", + ["provisioned-model/abcdefgh1234", "imported-model/abcdefgh1234", "custom-model-deployment/abcdefgh1234"], +) +def test_bare_opaque_arn_with_no_invoke_provider_routes_to_converse(arn_segment: str): """ - Only application-inference-profile ARNs are auto-routed to converse. Other - opaque ARNs (provisioned-model, imported-model, custom-model-deployment) - also yield no invoke provider, but they are frequently invoke-only with - provider-specific body formats, so routing them to converse could break - them. Guards the deliberate scope against an over-broad "any opaque ARN -> - converse" generalization. + A bare provisioned-model, imported-model, or custom-model-deployment ARN names no Invoke provider, + so Invoke could only fail locally with "Unknown provider=None". It goes to Converse instead, and a + caller who needs Invoke for one still names the provider (``bedrock/llama/arn:...``). """ - for arn_segment in ( - "provisioned-model/abcdefgh1234", - "imported-model/abcdefgh1234", - "custom-model-deployment/abcdefgh1234", - ): - route = BedrockModelInfo.get_bedrock_route( - model=f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}" - ) - assert route == "invoke", f"{arn_segment} should stay on invoke route" + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + model: Final = f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}" + assert BaseAWSLLM.get_bedrock_invoke_provider(model) is None + assert BedrockModelInfo.get_bedrock_route(model) == "converse" + assert BedrockModelInfo.get_bedrock_route(f"bedrock/llama/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}") == "invoke" def test_govcloud_cross_region_inference_prefix(): @@ -1110,12 +1108,11 @@ def test_unmapped_openai_family_model_routes_to_converse(): ], ) def test_unmapped_converse_only_family_model_routes_to_converse(unmapped: str): - """Same rule as the OpenAI family, for the other Bedrock-native families with no invoke handler. + """An id newer than the cost map from a family with no Invoke provider goes to Converse. minimax, nvidia, writer, zai, google and xai have no transformation under - ``litellm/llms/bedrock/chat/invoke_transformations``; every id of theirs is served by Converse. - An id newer than the cost map used to fall to the invoke route, where - ``get_bedrock_invoke_provider`` returns ``None`` and the request fails, instead of Converse. + ``litellm/llms/bedrock/chat/invoke_transformations``. Such an id used to fall to the invoke route, + where ``get_bedrock_invoke_provider`` returns ``None`` and the request fails before it is sent. """ import litellm @@ -1124,7 +1121,7 @@ def test_unmapped_converse_only_family_model_routes_to_converse(unmapped: str): def test_unmapped_model_of_family_with_invoke_handler_still_routes_to_invoke(): - """The converse-only default is scoped to families without an invoke handler; ``meta`` has one.""" + """The Converse default is scoped to ids with no Invoke provider, and ``meta`` has one.""" from typing import Final import litellm @@ -1134,6 +1131,59 @@ def test_unmapped_model_of_family_with_invoke_handler_still_routes_to_invoke(): assert BedrockModelInfo.get_bedrock_route(unmapped) == "invoke" +@pytest.mark.parametrize( + ("model", "model_path", "response_json"), + [ + ( + "bedrock/us.newvendor.model-1", + b"/model/us.newvendor.model-1/converse", + { + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + }, + ), + ( + "bedrock/mistral.mistral-99-unmapped-v1:0", + b"/model/mistral.mistral-99-unmapped-v1:0/invoke", + {"outputs": [{"text": "ok", "stop_reason": "stop"}]}, + ), + ], +) +def test_completion_on_unmapped_model_posts_converse_unless_its_id_names_an_invoke_provider( + model: str, model_path: bytes, response_json: dict[str, object] +): + """An id missing from the cost map used to be sent to Invoke, which fails locally for one naming no + Invoke provider (``Unknown provider=None``) before any request leaves, so a new vendor's model was unusable + without the ``converse/`` prefix.""" + import httpx + + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + assert BedrockModelInfo.get_base_model(model) not in litellm.bedrock_converse_models + sent: Final[list[httpx.Request]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response( + 200, + json=response_json, + headers={"x-amzn-bedrock-input-token-count": "1", "x-amzn-bedrock-output-token-count": "1"}, + ) + + response: Final = litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert response.choices[0].message.content == "ok" + assert [request.url.raw_path for request in sent] == [model_path] + + @pytest.mark.parametrize( ("model", "expected"), [ From 00ba727a8aef026fd81f27f8ec33c68b5e846c79 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:55:39 -0700 Subject: [PATCH 3/9] test(bedrock): expect Converse for ids with no Invoke provider in the nova ARN and not-found tests --- .../chat/test_converse_transformation.py | 21 +++++++++++++------ .../llms/bedrock/test_nova_imported_models.py | 6 ++---- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 154797bab5a..ab05a637a64 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -9184,17 +9184,26 @@ def test_bedrock_tools_transformation_valid_params(): def test_not_found_error(): + sent: Final[list[httpx.Request]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response( + 404, json={"message": "model not found"}, headers={"x-amzn-ErrorType": "ResourceNotFoundException"} + ) + with pytest.raises(litellm.NotFoundError): completion( model="bedrock/bad_model", - messages=[ - { - "role": "user", - "content": "What is the meaning of life", - } - ], + messages=[{"role": "user", "content": "What is the meaning of life"}], + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", ) + assert [request.url.raw_path for request in sent] == [b"/model/bad_model/converse"] + @pytest.mark.parametrize( "model, expected_base_model", diff --git a/tests/unit/llms/bedrock/test_nova_imported_models.py b/tests/unit/llms/bedrock/test_nova_imported_models.py index e3677aaf9e6..231f3097dcd 100644 --- a/tests/unit/llms/bedrock/test_nova_imported_models.py +++ b/tests/unit/llms/bedrock/test_nova_imported_models.py @@ -24,10 +24,8 @@ class TestGetBedrockRoute: def test_nova2_prefix_routes_to_converse(self): assert BedrockModelInfo.get_bedrock_route(NOVA2_MODEL) == "converse" - def test_plain_arn_routes_to_invoke(self): - # Without spec prefix, ARN doesn't match converse models - result = BedrockModelInfo.get_bedrock_route(f"bedrock/{NOVA_ARN}") - assert result == "invoke" + def test_plain_arn_without_an_invoke_provider_routes_to_converse(self): + assert BedrockModelInfo.get_bedrock_route(f"bedrock/{NOVA_ARN}") == "converse" class TestGetBedrockBaseModel: From 3ba4451f3be804581c346feae6ed35e9698b948b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:28:56 -0700 Subject: [PATCH 4/9] fix(bedrock): accept tools on Converse for a model with no cost-map row or Invoke provider, and keep a 404 for an invalid id --- .../exception_mapping_utils.py | 7 ++ .../bedrock/chat/converse_transformation.py | 16 +++- litellm/llms/bedrock/common_utils.py | 6 +- .../chat/test_converse_transformation.py | 77 ++++++++++++++++++- .../llms/bedrock/test_bedrock_common_utils.py | 10 --- 5 files changed, 100 insertions(+), 16 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 47ec8331c83..cdb6562c114 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -966,6 +966,13 @@ def _map_bedrock_exception( llm_provider="bedrock", response=getattr(original_exception, "response", None), ) + elif "The provided model identifier is invalid" in error_str: + raise NotFoundError( + message=f"BedrockException - {error_str}", + model=model, + llm_provider="bedrock", + response=getattr(original_exception, "response", None), + ) elif "Malformed input request" in error_str: raise BadRequestError( message=f"BedrockException - {error_str}", diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index f54c352fd6e..ed1c0623e9d 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -116,12 +116,14 @@ from litellm.utils import ( token_counter, ) +from ..base_aws_llm import BaseAWSLLM from ..common_utils import ( BedrockError, BedrockModelInfo, bedrock_cache_point_ttl_supported, bedrock_capability_model, bedrock_converse_supports_parallel_tool_use_config, + bedrock_cost_map_has_row, bedrock_model_accepts_cache_points, bedrock_reasoning_effort_disabled, bedrock_rejects_stop_sequences, @@ -660,6 +662,13 @@ class AmazonConverseConfig(BaseConfig): return flag return True + @staticmethod + def _has_no_cost_map_row_or_invoke_provider(model: str, base_model: str) -> bool: + candidates: Final = (model, base_model, *(f"{prefix}{base_model}" for prefix in ("global.", "us.", "eu."))) + if bedrock_cost_map_has_row(candidates): + return False + return BaseAWSLLM.get_bedrock_invoke_provider(model) is None + def get_supported_openai_params(self, model: str) -> list[str]: from litellm.utils import supports_function_calling @@ -678,7 +687,9 @@ class AmazonConverseConfig(BaseConfig): "parallel_tool_calls", ] - if "arn" in model: # we can't infer the model from the arn, so just add all params + base_model: Final = BedrockModelInfo.get_base_model(model) + + if "arn" in model or self._has_no_cost_map_row_or_invoke_provider(model=model, base_model=base_model): supported_params.append("tools") supported_params.append("tool_choice") supported_params.append("thinking") @@ -689,9 +700,6 @@ class AmazonConverseConfig(BaseConfig): supported_params.append("web_search_options") return supported_params - ## Filter out 'cross-region' from model name - base_model: Final = BedrockModelInfo.get_base_model(model) - if ( base_model.startswith("anthropic") or base_model.startswith("mistral") diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index eaa5ff21c9a..07c81ea532a 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -10,7 +10,7 @@ import json import os import re import warnings -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias @@ -1092,6 +1092,10 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None: return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw) +def bedrock_cost_map_has_row(keys: Iterable[str]) -> bool: + return any(_model_cost_entry(key) is not None for key in keys) + + def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]: return tuple( _model_cost_entry(key) diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index ab05a637a64..79731629e50 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -9189,7 +9189,9 @@ def test_not_found_error(): def respond(request: httpx.Request) -> httpx.Response: sent.append(request) return httpx.Response( - 404, json={"message": "model not found"}, headers={"x-amzn-ErrorType": "ResourceNotFoundException"} + 400, + json={"message": "The provided model identifier is invalid."}, + headers={"x-amzn-ErrorType": "ValidationException"}, ) with pytest.raises(litellm.NotFoundError): @@ -9205,6 +9207,79 @@ def test_not_found_error(): assert [request.url.raw_path for request in sent] == [b"/model/bad_model/converse"] +_WEATHER_TOOL: Final = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather for a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, +} + + +def _converse_tool_call_reply(sent: list[httpx.Request]): + def respond(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response( + 200, + json={ + "output": { + "message": { + "role": "assistant", + "content": [{"toolUse": {"toolUseId": "t1", "name": "get_weather", "input": {"city": "Paris"}}}], + } + }, + "stopReason": "tool_use", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + }, + ) + + return respond + + +def test_model_missing_from_cost_map_sends_tools_to_converse(): + model: Final = "bedrock/converse/us.newvendor.model-1" + assert "us.newvendor.model-1" not in litellm.model_cost + sent: Final[list[httpx.Request]] = [] + + response: Final = completion( + model=model, + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + tool_choice="auto", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert response.choices[0].message.tool_calls[0].function.name == "get_weather" + assert [request.url.raw_path for request in sent] == [b"/model/us.newvendor.model-1/converse"] + tool_config: Final = json.loads(sent[0].content)["toolConfig"] + assert [tool["toolSpec"]["name"] for tool in tool_config["tools"]] == ["get_weather"] + assert tool_config["toolChoice"] == {"auto": {}} + + +def test_mapped_model_without_function_calling_still_rejects_tools(): + model: Final = "bedrock/converse/meta.llama3-8b-instruct-v1:0" + assert "meta.llama3-8b-instruct-v1:0" in litellm.model_cost + assert not litellm.supports_function_calling(model=model) + sent: Final[list[httpx.Request]] = [] + + with pytest.raises(litellm.UnsupportedParamsError): + completion( + model=model, + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert sent == [] + + @pytest.mark.parametrize( "model, expected_base_model", [ diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index f98de8d1c1e..f57b0e57a75 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -1108,12 +1108,6 @@ def test_unmapped_openai_family_model_routes_to_converse(): ], ) def test_unmapped_converse_only_family_model_routes_to_converse(unmapped: str): - """An id newer than the cost map from a family with no Invoke provider goes to Converse. - - minimax, nvidia, writer, zai, google and xai have no transformation under - ``litellm/llms/bedrock/chat/invoke_transformations``. Such an id used to fall to the invoke route, - where ``get_bedrock_invoke_provider`` returns ``None`` and the request fails before it is sent. - """ import litellm assert BedrockModelInfo.get_base_model(unmapped) not in litellm.bedrock_converse_models @@ -1121,7 +1115,6 @@ def test_unmapped_converse_only_family_model_routes_to_converse(unmapped: str): def test_unmapped_model_of_family_with_invoke_handler_still_routes_to_invoke(): - """The Converse default is scoped to ids with no Invoke provider, and ``meta`` has one.""" from typing import Final import litellm @@ -1153,9 +1146,6 @@ def test_unmapped_model_of_family_with_invoke_handler_still_routes_to_invoke(): def test_completion_on_unmapped_model_posts_converse_unless_its_id_names_an_invoke_provider( model: str, model_path: bytes, response_json: dict[str, object] ): - """An id missing from the cost map used to be sent to Invoke, which fails locally for one naming no - Invoke provider (``Unknown provider=None``) before any request leaves, so a new vendor's model was unusable - without the ``converse/`` prefix.""" import httpx from litellm.llms.custom_httpx.http_handler import HTTPHandler From 4875442684f347c40b8076b7213bd34bb0565240 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:10:43 -0700 Subject: [PATCH 5/9] fix(bedrock): answer 404 for an invalid model id instead of AWS's 400 AWS answers a mistyped Bedrock model id with a 400 ValidationException. The mapping already raised NotFoundError, but passing AWS's 400 response through made the openai base class report 400, so the proxy answered 400 and the router skipped its cooldown --- litellm/litellm_core_utils/exception_mapping_utils.py | 1 - tests/unit/llms/bedrock/chat/test_converse_transformation.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index cdb6562c114..ea225d542cd 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -971,7 +971,6 @@ def _map_bedrock_exception( message=f"BedrockException - {error_str}", model=model, llm_provider="bedrock", - response=getattr(original_exception, "response", None), ) elif "Malformed input request" in error_str: raise BadRequestError( diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 79731629e50..a49cd628c88 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -9194,7 +9194,7 @@ def test_not_found_error(): headers={"x-amzn-ErrorType": "ValidationException"}, ) - with pytest.raises(litellm.NotFoundError): + with pytest.raises(litellm.NotFoundError) as exc_info: completion( model="bedrock/bad_model", messages=[{"role": "user", "content": "What is the meaning of life"}], @@ -9204,6 +9204,7 @@ def test_not_found_error(): aws_region_name="us-west-2", ) + assert exc_info.value.status_code == 404 assert [request.url.raw_path for request in sent] == [b"/model/bad_model/converse"] From f09bf3ac356140d3950a7a853127a950b8657845 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:20:00 +0000 Subject: [PATCH 6/9] fix(bedrock): drop lookaround regexes from tool schemas for a model the cost map does not list --- litellm/llms/bedrock/common_utils.py | 21 +++++-- .../chat/test_converse_transformation.py | 60 +++++++++++++++++++ 2 files changed, 76 insertions(+), 5 deletions(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 07c81ea532a..31452e7e4e7 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1559,7 +1559,8 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[ The deployment's ``model_info.supports_regex_lookaround`` wins, then the ``model_prices_and_context_window.json`` entry of its ``base_model``, then the - entry of ``model`` itself. A model nobody flagged keeps its schema as sent. + entry of ``model`` itself. A listed model nobody flagged keeps its schema as sent, + and a model the cost map does not list drops the lookarounds. """ params: Final = litellm_params or {} deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround") @@ -1568,7 +1569,10 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[ base_model: Final = params.get("base_model") candidates: Final = (*((base_model,) if isinstance(base_model, str) else ()), model) flags: Final = (_bedrock_converse_model_flag(candidate, "supports_regex_lookaround") for candidate in candidates) - return next((flag for flag in flags if flag is not None), True) + flag: Final = next((flag for flag in flags if flag is not None), None) + if flag is not None: + return flag + return any(_bedrock_cost_map_lists(candidate) for candidate in candidates) _BedrockConverseModelFlag: TypeAlias = Literal[ @@ -1577,10 +1581,17 @@ _BedrockConverseModelFlag: TypeAlias = Literal[ ] -def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None: +def _bedrock_model_info_candidates(model: str) -> tuple[str, ...]: base: Final = get_bedrock_base_model(model) - candidates: Final = dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base))) - for candidate in candidates: + return tuple(dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base)))) + + +def _bedrock_cost_map_lists(model: str) -> bool: + return bedrock_cost_map_has_row(_bedrock_model_info_candidates(model)) + + +def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None: + for candidate in _bedrock_model_info_candidates(model): with contextlib.suppress(Exception): model_info = get_cached_model_info()( model=candidate, diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index a49cd628c88..285941c97d3 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -1214,6 +1214,66 @@ def test_transform_request_drops_lookaround_regex_when_the_deployment_model_info assert properties == _LOOKAROUND_FREE_PROPERTIES +@pytest.mark.parametrize( + "model", + [ + "us.newvendor.model-1", + "arn:aws:bedrock:us-east-1:123456789012:custom-model-deployment/abc123", + ], +) +def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list(model): + assert model not in litellm.model_cost + + assert _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL) == _LOOKAROUND_FREE_PROPERTIES + + +def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_whose_deployment_opts_out(): + properties = _tool_schema_properties( + "us.newvendor.model-1", _ARTIFACT_DATA_ANTHROPIC_TOOL, {"model_info": {"supports_regex_lookaround": True}} + ) + + assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"] + + +def test_a_router_deployment_of_a_model_the_cost_map_does_not_list_drops_lookaround_regex(respx_mock): + route: Final = respx_mock.post( + "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.newvendor.model-1/converse" + ).mock( + return_value=httpx.Response( + 200, + json={ + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + }, + ) + ) + router: Final = Router( + model_list=[ + { + "model_name": "unlisted", + "litellm_params": { + "model": "bedrock/us.newvendor.model-1", + "aws_access_key_id": "fake-access-key", + "aws_secret_access_key": "fake-secret-key", + "aws_region_name": "us-east-1", + }, + "model_info": {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}, + } + ] + ) + + router.completion( + model="unlisted", + messages=[{"role": "user", "content": "hi"}], + tools=[_ARTIFACT_DATA_OPENAI_TOOL], + client=HTTPHandler(), + ) + + tool: Final = json.loads(route.calls[0].request.content)["toolConfig"]["tools"][0] + assert tool["toolSpec"]["inputSchema"]["json"]["properties"] == _LOOKAROUND_FREE_PROPERTIES + + def test_transform_request_keeps_lookaround_regex_when_the_deployment_model_info_opts_out(): properties = _tool_schema_properties( "global.moonshotai.kimi-k3", _ARTIFACT_DATA_ANTHROPIC_TOOL, {"model_info": {"supports_regex_lookaround": True}} From dc903ab3bc1a6d97809c92f573f8b589af91ac8e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:50:00 +0000 Subject: [PATCH 7/9] fix(bedrock): apply the unknown-model defaults only to bare ids LiteLLM knows nothing about A bare Bedrock id with no cost-map row and no Invoke vendor gets Converse's open params, its reasoning_effort as sent, lookaround-free tool schemas, and a 404 for AWS's invalid-identifier reply, which keeps AWS's response headers. An explicit route like converse/, an ARN, or an id LiteLLM recognizes keeps main's params, regexes and error status, so drop_params still drops tools there --- .../exception_mapping_utils.py | 16 ++- .../bedrock/chat/converse_transformation.py | 22 ++-- litellm/llms/bedrock/common_utils.py | 35 ++++-- .../test_exception_mapping_utils.py | 19 ++- .../chat/test_converse_transformation.py | 109 +++++++++++++++--- 5 files changed, 155 insertions(+), 46 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index ea225d542cd..d4e88ce6309 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import ( ) from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string +from litellm.llms.bedrock.common_utils import bedrock_model_is_unknown from litellm.types.utils import LlmProviders from ..exceptions import ( @@ -901,13 +902,15 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE: Final = ( ) -def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPException) -> httpx.Response: +def _bedrock_response_with_status(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response: original_response: Final = getattr(original_exception, "response", None) if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None: - return httpx.Response(status_code=400, request=httpx.Request(method="POST", url="https://api.openai.com/v1/")) - if original_response.status_code == 400: + return httpx.Response( + status_code=status_code, request=httpx.Request(method="POST", url="https://api.openai.com/v1/") + ) + if original_response.status_code == status_code: return original_response - return httpx.Response(status_code=400, headers=original_response.headers, request=original_response.request) + return httpx.Response(status_code=status_code, headers=original_response.headers, request=original_response.request) def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: @@ -943,7 +946,7 @@ def _map_bedrock_exception( message=mantle_context_window_message, model=model, llm_provider=custom_llm_provider, - response=_bedrock_mantle_context_window_response(original_exception), + response=_bedrock_response_with_status(original_exception, 400), ) if ( "too many tokens" in error_str @@ -966,11 +969,12 @@ def _map_bedrock_exception( llm_provider="bedrock", response=getattr(original_exception, "response", None), ) - elif "The provided model identifier is invalid" in error_str: + elif "The provided model identifier is invalid" in error_str and bedrock_model_is_unknown(model): raise NotFoundError( message=f"BedrockException - {error_str}", model=model, llm_provider="bedrock", + response=_bedrock_response_with_status(original_exception, 404), ) elif "Malformed input request" in error_str: raise BadRequestError( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index ed1c0623e9d..9eb5d3433d3 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -61,6 +61,7 @@ from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_image from litellm.llms.bedrock.common_utils import ( bedrock_chat_uses_native_structured_output, bedrock_cleaned_output_schema, + bedrock_model_is_unknown, bedrock_model_supports_regex_lookaround, bedrock_native_output_format, bedrock_native_output_is_opt_in, @@ -116,14 +117,12 @@ from litellm.utils import ( token_counter, ) -from ..base_aws_llm import BaseAWSLLM from ..common_utils import ( BedrockError, BedrockModelInfo, bedrock_cache_point_ttl_supported, bedrock_capability_model, bedrock_converse_supports_parallel_tool_use_config, - bedrock_cost_map_has_row, bedrock_model_accepts_cache_points, bedrock_reasoning_effort_disabled, bedrock_rejects_stop_sequences, @@ -528,13 +527,14 @@ class AmazonConverseConfig(BaseConfig): """ Handle the reasoning_effort parameter based on the model type. - - GPT-OSS and DeepSeek V3 models: passed through unchanged via additionalModelRequestFields. + - GPT-OSS, DeepSeek V3 and unknown models (``bedrock_model_is_unknown``): passed through unchanged + via additionalModelRequestFields. - OpenAI GPT-5.x and GPT-6 models: mapped to ``reasoning.effort`` via additionalModelRequestFields. - Nova 2 models: transformed to reasoningConfig. - Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on adaptive Claude 4.6 / 4.7). """ - if "gpt-oss" in model or "deepseek" in model: + if "gpt-oss" in model or "deepseek" in model or bedrock_model_is_unknown(model): optional_params["reasoning_effort"] = reasoning_effort elif self._is_openai_gpt_reasoning_model(model): reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort} @@ -662,13 +662,6 @@ class AmazonConverseConfig(BaseConfig): return flag return True - @staticmethod - def _has_no_cost_map_row_or_invoke_provider(model: str, base_model: str) -> bool: - candidates: Final = (model, base_model, *(f"{prefix}{base_model}" for prefix in ("global.", "us.", "eu."))) - if bedrock_cost_map_has_row(candidates): - return False - return BaseAWSLLM.get_bedrock_invoke_provider(model) is None - def get_supported_openai_params(self, model: str) -> list[str]: from litellm.utils import supports_function_calling @@ -687,9 +680,7 @@ class AmazonConverseConfig(BaseConfig): "parallel_tool_calls", ] - base_model: Final = BedrockModelInfo.get_base_model(model) - - if "arn" in model or self._has_no_cost_map_row_or_invoke_provider(model=model, base_model=base_model): + if "arn" in model or bedrock_model_is_unknown(model): supported_params.append("tools") supported_params.append("tool_choice") supported_params.append("thinking") @@ -700,6 +691,9 @@ class AmazonConverseConfig(BaseConfig): supported_params.append("web_search_options") return supported_params + ## Filter out 'cross-region' from model name + base_model: Final = BedrockModelInfo.get_base_model(model) + if ( base_model.startswith("anthropic") or base_model.startswith("mistral") diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 31452e7e4e7..e8bb1cfd4cb 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -10,7 +10,7 @@ import json import os import re import warnings -from collections.abc import Iterable, Iterator, Mapping, Sequence +from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias @@ -1092,10 +1092,6 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None: return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw) -def bedrock_cost_map_has_row(keys: Iterable[str]) -> bool: - return any(_model_cost_entry(key) is not None for key in keys) - - def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]: return tuple( _model_cost_entry(key) @@ -1559,8 +1555,8 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[ The deployment's ``model_info.supports_regex_lookaround`` wins, then the ``model_prices_and_context_window.json`` entry of its ``base_model``, then the - entry of ``model`` itself. A listed model nobody flagged keeps its schema as sent, - and a model the cost map does not list drops the lookarounds. + entry of ``model`` itself. A model nobody flagged keeps its schema as sent, unless + every candidate is a bare id that ``bedrock_model_is_unknown`` describes. """ params: Final = litellm_params or {} deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround") @@ -1572,7 +1568,26 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[ flag: Final = next((flag for flag in flags if flag is not None), None) if flag is not None: return flag - return any(_bedrock_cost_map_lists(candidate) for candidate in candidates) + return not all(bedrock_model_is_unknown(candidate) for candidate in candidates) + + +def bedrock_model_is_unknown(model: str) -> bool: + """ + Whether ``model`` is a bare Bedrock model id that no cost-map row describes and whose vendor + has no Invoke handler, so it routes to Converse with nothing to say what the model accepts. + + An id carrying a route or provider prefix, a region path, or an ARN is never unknown. + """ + model_id: Final = model.removeprefix("bedrock/") + if "/" in model_id: + return False + base: Final = get_bedrock_base_model(model_id) + regional_ids: Final = (f"{region}.{base}" for region in get_bedrock_cross_region_inference_regions()) + if any(_model_cost_entry(key) is not None for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)): + return False + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + return BaseAWSLLM.get_bedrock_invoke_provider(model_id) is None _BedrockConverseModelFlag: TypeAlias = Literal[ @@ -1586,10 +1601,6 @@ def _bedrock_model_info_candidates(model: str) -> tuple[str, ...]: return tuple(dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base)))) -def _bedrock_cost_map_lists(model: str) -> bool: - return bedrock_cost_map_has_row(_bedrock_model_info_candidates(model)) - - def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None: for candidate in _bedrock_model_info_candidates(model): with contextlib.suppress(Exception): diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 55e7c47e2c9..f8e753ae4e7 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -58,7 +58,6 @@ def test_openai_cyber_policy_uses_the_structured_error_code(shape: str, code: st assert caught.value.response.headers["x-request-id"] == "cyber-policy-request" - def _debug_info_for_api_base(configured_api_base: str) -> str: request: Final = httpx.Request("POST", "https://gateway.example/v1/chat/completions") original: Final = OpenAIError( @@ -1468,6 +1467,24 @@ def test_bedrock_mantle_400_maps_to_bad_request(): assert type(excinfo.value) is litellm.BadRequestError +class _ErrorWithoutResponse(Exception): + def __init__(self, message: str, status_code: int) -> None: + super().__init__(message) + self.message = message + self.status_code = status_code + + +def test_bedrock_invalid_model_identifier_without_a_provider_response_still_answers_404(): + with pytest.raises(litellm.NotFoundError) as excinfo: + exception_type( + model="us.newvendor.typo-1", + original_exception=_ErrorWithoutResponse("The provided model identifier is invalid.", 400), + custom_llm_provider="bedrock", + ) + + assert excinfo.value.response.status_code == 404 + + def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded(): from litellm.llms.base_llm.chat.transformation import BaseLLMException diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 285941c97d3..40b0511582a 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -1214,17 +1214,29 @@ def test_transform_request_drops_lookaround_regex_when_the_deployment_model_info assert properties == _LOOKAROUND_FREE_PROPERTIES +def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list(): + assert "us.newvendor.model-1" not in litellm.model_cost + + assert _tool_schema_properties("us.newvendor.model-1", _ARTIFACT_DATA_ANTHROPIC_TOOL) == _LOOKAROUND_FREE_PROPERTIES + + @pytest.mark.parametrize( "model", [ - "us.newvendor.model-1", + "converse/us.newvendor.model-1", "arn:aws:bedrock:us-east-1:123456789012:custom-model-deployment/abc123", + "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", + "us.anthropic.claude-unreleased-v1:0", + "eu.openai.gpt-unreleased", + "apac.newvendor.listed-1", ], ) -def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list(model): - assert model not in litellm.model_cost +def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_litellm_still_recognizes(model, monkeypatch): + monkeypatch.setitem(litellm.model_cost, "us.newvendor.listed-1", {"litellm_provider": "bedrock_converse"}) - assert _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL) == _LOOKAROUND_FREE_PROPERTIES + properties = _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL) + + assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"] def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_whose_deployment_opts_out(): @@ -5408,7 +5420,6 @@ def test_recursive_output_config_format_follows_the_profile_native_key( assert ("outputConfig" in request) is forwarded - def test_sonnet_4_6_json_schema_still_uses_native_output_config(monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) @@ -9243,31 +9254,65 @@ def test_bedrock_tools_transformation_valid_params(): assert "test" in result[0]["toolSpec"]["inputSchema"]["json"]["required"] -def test_not_found_error(): - sent: Final[list[httpx.Request]] = [] - +def _invalid_model_identifier_reply(sent: list[httpx.Request]): def respond(request: httpx.Request) -> httpx.Response: sent.append(request) return httpx.Response( 400, json={"message": "The provided model identifier is invalid."}, - headers={"x-amzn-ErrorType": "ValidationException"}, + headers={"x-amzn-ErrorType": "ValidationException", "x-amzn-RequestId": "req-123"}, ) + return respond + + +def test_not_found_error(): + sent: Final[list[httpx.Request]] = [] + with pytest.raises(litellm.NotFoundError) as exc_info: completion( model="bedrock/bad_model", messages=[{"role": "user", "content": "What is the meaning of life"}], - client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), + client=HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent))) + ), aws_access_key_id="testing", aws_secret_access_key="testing", aws_region_name="us-west-2", ) assert exc_info.value.status_code == 404 + assert exc_info.value.response.status_code == 404 + assert exc_info.value.response.headers["x-amzn-requestid"] == "req-123" assert [request.url.raw_path for request in sent] == [b"/model/bad_model/converse"] +@pytest.mark.parametrize( + "model, path", + [ + ("bedrock/converse/bad_model", b"/model/bad_model/converse"), + ("bedrock/anthropic.claude-typo-v1:0", b"/model/anthropic.claude-typo-v1:0/invoke"), + ], +) +def test_invalid_model_identifier_stays_a_bad_request_for_a_model_litellm_recognizes(model, path): + sent: Final[list[httpx.Request]] = [] + + with pytest.raises(litellm.BadRequestError) as exc_info: + completion( + model=model, + messages=[{"role": "user", "content": "What is the meaning of life"}], + client=HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent))) + ), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert exc_info.value.status_code == 400 + assert [request.url.raw_path for request in sent] == [path] + + _WEATHER_TOOL: Final = { "type": "function", "function": { @@ -9287,7 +9332,9 @@ def _converse_tool_call_reply(sent: list[httpx.Request]): "output": { "message": { "role": "assistant", - "content": [{"toolUse": {"toolUseId": "t1", "name": "get_weather", "input": {"city": "Paris"}}}], + "content": [ + {"toolUse": {"toolUseId": "t1", "name": "get_weather", "input": {"city": "Paris"}}} + ], } }, "stopReason": "tool_use", @@ -9299,7 +9346,7 @@ def _converse_tool_call_reply(sent: list[httpx.Request]): def test_model_missing_from_cost_map_sends_tools_to_converse(): - model: Final = "bedrock/converse/us.newvendor.model-1" + model: Final = "bedrock/us.newvendor.model-1" assert "us.newvendor.model-1" not in litellm.model_cost sent: Final[list[httpx.Request]] = [] @@ -9321,6 +9368,39 @@ def test_model_missing_from_cost_map_sends_tools_to_converse(): assert tool_config["toolChoice"] == {"auto": {}} +def test_explicit_converse_route_to_an_unlisted_model_still_drops_tools_under_drop_params(): + sent: Final[list[httpx.Request]] = [] + + completion( + model="bedrock/converse/us.newvendor.model-1", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + drop_params=True, + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert "toolConfig" not in json.loads(sent[0].content) + + +def test_unknown_model_gets_reasoning_effort_as_sent(): + sent: Final[list[httpx.Request]] = [] + + completion( + model="bedrock/us.newvendor.model-1", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + reasoning_effort="high", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert json.loads(sent[0].content)["additionalModelRequestFields"] == {"reasoning_effort": "high"} + + def test_mapped_model_without_function_calling_still_rejects_tools(): model: Final = "bedrock/converse/meta.llama3-8b-instruct-v1:0" assert "meta.llama3-8b-instruct-v1:0" in litellm.model_cost @@ -10810,7 +10890,10 @@ def test_uses_opt_in_native_output_reads_the_base_model_of_an_application_infere _CALLER_OUTPUT_CONFIG_WITH_A_SCHEMA_OBJECT: Final = { - "textFormat": {"type": "json_schema", "structure": {"jsonSchema": {"name": "capital", "schema": {"type": "object"}}}} + "textFormat": { + "type": "json_schema", + "structure": {"jsonSchema": {"name": "capital", "schema": {"type": "object"}}}, + } } _BEDROCK_SCHEMA_TYPE_ERROR: Final = "outputConfig.textFormat.structure.jsonSchema.schema must be a string" From 1f2ca49601ac7d763166e1feed8cfc3da6084cb4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:35:09 -0700 Subject: [PATCH 8/9] fix(bedrock): judge lookaround support by model id on every route, ignore price-only rows The regex check now strips route and region prefixes from the model and its base_model before asking whether LiteLLM knows the model, so bedrock/converse/ gets the same answer in a direct transform call as on the completion path. A cost-map row counts only when it names a provider, so a price-only register_model row leaves an unlisted id unknown, as on the proxy. AmazonInvokeConfig now inherits the Invoke provider lookup instead of keeping an identical copy. --- .../base_invoke_transformation.py | 62 +------------------ litellm/llms/bedrock/common_utils.py | 17 +++-- .../chat/test_converse_transformation.py | 49 +++++++++++++-- .../llms/bedrock/test_bedrock_common_utils.py | 5 -- 4 files changed, 53 insertions(+), 80 deletions(-) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 852179e8caa..877fb4ca692 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -2,7 +2,7 @@ import copy import json import time from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, cast, get_args +from typing import TYPE_CHECKING, Any, Final import httpx from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config @@ -74,10 +74,6 @@ _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = ( "{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}" ) -_INVOKE_PROVIDERS: Final = TypeAdapter(tuple[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, ...]).validate_python( - get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL) -) - def _response_header(headers: Mapping[str, str], name: str) -> str | None: return headers.get(name, None) @@ -562,62 +558,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): """ return False - @staticmethod - def get_bedrock_invoke_provider( - model: str, - ) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None: - """ - Helper function to get the bedrock provider from the model - - handles 4 scenarios: - 1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic` - 2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic` - 3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama` - 4. model=us.amazon.nova-pro-v1:0 -> Returns `nova` - """ - if model.startswith("invoke/"): - model = model.replace("invoke/", "", 1) - - # Special case: Check for "nova" in model name first (before "amazon") - # This handles amazon.nova-* models which would otherwise match "amazon" (Titan) - if "nova" in model.lower(): - if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): - return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova") - - _split_model: Final = model.split(".")[0] - if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): - return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model) - - # If not a known provider, check for pattern with two slashes - provider = AmazonInvokeConfig._get_provider_from_model_path(model) - if provider is not None: - return provider - - for provider in _INVOKE_PROVIDERS: - if provider in model: - return provider - return None - - @staticmethod - def _get_provider_from_model_path( - model_path: str, - ) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None: - """ - Helper function to get the provider from a model path with format: provider/model-name - - Args: - model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name') - - Returns: - Optional[str]: The provider name, or None if no valid provider found - """ - parts: Final = model_path.split("/") - if len(parts) >= 1: - provider: Final = parts[0] - if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): - return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider) - return None - def convert_messages_to_prompt(self, model: str, messages, provider, custom_prompt_dict) -> tuple[str, list | None]: # handle anthropic prompts and amazon titan prompts prompt = "" diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 87d815eb8c1..1e9b3ca2d0a 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1109,6 +1109,11 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None: return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw) +def _cost_map_row_names_a_provider(key: str) -> bool: + entry: Final = _model_cost_entry(key) + return entry is not None and entry.get("litellm_provider") is not None + + def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]: return tuple( _model_cost_entry(key) @@ -1579,7 +1584,7 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[ The deployment's ``model_info.supports_regex_lookaround`` wins, then the ``model_prices_and_context_window.json`` entry of its ``base_model``, then the entry of ``model`` itself. A model nobody flagged keeps its schema as sent, unless - every candidate is a bare id that ``bedrock_model_is_unknown`` describes. + no candidate names a model LiteLLM knows, whatever route prefix it carries. """ params: Final = litellm_params or {} deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround") @@ -1591,22 +1596,16 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[ flag: Final = next((flag for flag in flags if flag is not None), None) if flag is not None: return flag - return not all(bedrock_model_is_unknown(candidate) for candidate in candidates) + return not all(bedrock_model_is_unknown(split_bedrock_region_path(candidate)[1]) for candidate in candidates) def bedrock_model_is_unknown(model: str) -> bool: - """ - Whether ``model`` is a bare Bedrock model id that no cost-map row describes and whose vendor - has no Invoke handler, so it routes to Converse with nothing to say what the model accepts. - - An id carrying a route or provider prefix, a region path, or an ARN is never unknown. - """ model_id: Final = model.removeprefix("bedrock/") if "/" in model_id: return False base: Final = get_bedrock_base_model(model_id) regional_ids: Final = (f"{region}.{base}" for region in get_bedrock_cross_region_inference_regions()) - if any(_model_cost_entry(key) is not None for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)): + if any(_cost_map_row_names_a_provider(key) for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)): return False from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 355d01f86f8..5a6f9422362 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -1221,16 +1221,26 @@ def test_transform_request_drops_lookaround_regex_when_the_deployment_model_info assert properties == _LOOKAROUND_FREE_PROPERTIES -def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list(): +@pytest.mark.parametrize( + ("model", "litellm_params"), + [ + ("us.newvendor.model-1", None), + ("converse/us.newvendor.model-1", None), + ("us-east-1/us.newvendor.model-1", None), + ("us.newvendor.model-1", {"base_model": "bedrock/converse/us.newvendor.model-1"}), + ], +) +def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list(model, litellm_params): assert "us.newvendor.model-1" not in litellm.model_cost - assert _tool_schema_properties("us.newvendor.model-1", _ARTIFACT_DATA_ANTHROPIC_TOOL) == _LOOKAROUND_FREE_PROPERTIES + properties: Final = _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL, litellm_params) + + assert properties == _LOOKAROUND_FREE_PROPERTIES @pytest.mark.parametrize( "model", [ - "converse/us.newvendor.model-1", "arn:aws:bedrock:us-east-1:123456789012:custom-model-deployment/abc123", "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", "us.anthropic.claude-unreleased-v1:0", @@ -1254,7 +1264,16 @@ def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_whose_de assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"] -def test_a_router_deployment_of_a_model_the_cost_map_does_not_list_drops_lookaround_regex(respx_mock): +@pytest.mark.parametrize( + "deployment_params", + [ + {"model": "bedrock/us.newvendor.model-1"}, + {"model": "bedrock/converse/us.newvendor.model-1", "allowed_openai_params": ["tools"]}, + ], +) +def test_a_router_deployment_of_a_model_the_cost_map_does_not_list_drops_lookaround_regex( + respx_mock, deployment_params +): route: Final = respx_mock.post( "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.newvendor.model-1/converse" ).mock( @@ -1272,7 +1291,7 @@ def test_a_router_deployment_of_a_model_the_cost_map_does_not_list_drops_lookaro { "model_name": "unlisted", "litellm_params": { - "model": "bedrock/us.newvendor.model-1", + **deployment_params, "aws_access_key_id": "fake-access-key", "aws_secret_access_key": "fake-secret-key", "aws_region_name": "us-east-1", @@ -9375,6 +9394,26 @@ def test_model_missing_from_cost_map_sends_tools_to_converse(): assert tool_config["toolChoice"] == {"auto": {}} +def test_a_price_only_row_for_an_unlisted_model_still_sends_tools_to_converse(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, "us.newvendor.priced-1", {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6} + ) + sent: Final[list[httpx.Request]] = [] + + completion( + model="bedrock/us.newvendor.priced-1", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + tool_config: Final = json.loads(sent[0].content)["toolConfig"] + assert [tool["toolSpec"]["name"] for tool in tool_config["tools"]] == ["get_weather"] + + def test_explicit_converse_route_to_an_unlisted_model_still_drops_tools_under_drop_params(): sent: Final[list[httpx.Request]] = [] diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index f57b0e57a75..20c0e10cb81 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -219,11 +219,6 @@ def test_system_defined_inference_profile_arn_still_routes_to_converse(): ["provisioned-model/abcdefgh1234", "imported-model/abcdefgh1234", "custom-model-deployment/abcdefgh1234"], ) def test_bare_opaque_arn_with_no_invoke_provider_routes_to_converse(arn_segment: str): - """ - A bare provisioned-model, imported-model, or custom-model-deployment ARN names no Invoke provider, - so Invoke could only fail locally with "Unknown provider=None". It goes to Converse instead, and a - caller who needs Invoke for one still names the provider (``bedrock/llama/arn:...``). - """ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM model: Final = f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}" From fa9cd8fed386d42caf523073d9624c40fdfbce15 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:20:54 -0700 Subject: [PATCH 9/9] fix(bedrock): treat a region path like the bare id when the cost map does not list it A deployment like bedrock/us-east-1/ now gets the same tools, reasoning_effort and typo 404 as bedrock/, so Claude Code works on it and a mistyped one still answers 404. --- litellm/llms/bedrock/common_utils.py | 4 +++- .../llms/bedrock/chat/test_converse_transformation.py | 9 +++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 1e9b3ca2d0a..c744b6eff37 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1600,7 +1600,9 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[ def bedrock_model_is_unknown(model: str) -> bool: - model_id: Final = model.removeprefix("bedrock/") + routed_id: Final = model.removeprefix("bedrock/") + region, separator, regional_id = routed_id.partition("/") + model_id: Final = regional_id if separator and region in get_all_bedrock_regions() else routed_id if "/" in model_id: return False base: Final = get_bedrock_base_model(model_id) diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 5a6f9422362..7d47ca8bcdc 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -9292,12 +9292,13 @@ def _invalid_model_identifier_reply(sent: list[httpx.Request]): return respond -def test_not_found_error(): +@pytest.mark.parametrize("model", ["bedrock/bad_model", "bedrock/us-east-1/bad_model"]) +def test_not_found_error(model): sent: Final[list[httpx.Request]] = [] with pytest.raises(litellm.NotFoundError) as exc_info: completion( - model="bedrock/bad_model", + model=model, messages=[{"role": "user", "content": "What is the meaning of life"}], client=HTTPHandler( client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent))) @@ -9371,8 +9372,8 @@ def _converse_tool_call_reply(sent: list[httpx.Request]): return respond -def test_model_missing_from_cost_map_sends_tools_to_converse(): - model: Final = "bedrock/us.newvendor.model-1" +@pytest.mark.parametrize("model", ["bedrock/us.newvendor.model-1", "bedrock/us-east-1/us.newvendor.model-1"]) +def test_model_missing_from_cost_map_sends_tools_to_converse(model): assert "us.newvendor.model-1" not in litellm.model_cost sent: Final[list[httpx.Request]] = []