From 1f2ca49601ac7d763166e1feed8cfc3da6084cb4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:35:09 -0700 Subject: [PATCH] fix(bedrock): judge lookaround support by model id on every route, ignore price-only rows The regex check now strips route and region prefixes from the model and its base_model before asking whether LiteLLM knows the model, so bedrock/converse/ gets the same answer in a direct transform call as on the completion path. A cost-map row counts only when it names a provider, so a price-only register_model row leaves an unlisted id unknown, as on the proxy. AmazonInvokeConfig now inherits the Invoke provider lookup instead of keeping an identical copy. --- .../base_invoke_transformation.py | 62 +------------------ litellm/llms/bedrock/common_utils.py | 17 +++-- .../chat/test_converse_transformation.py | 49 +++++++++++++-- .../llms/bedrock/test_bedrock_common_utils.py | 5 -- 4 files changed, 53 insertions(+), 80 deletions(-) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 852179e8caa..877fb4ca692 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -2,7 +2,7 @@ import copy import json import time from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, cast, get_args +from typing import TYPE_CHECKING, Any, Final import httpx from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config @@ -74,10 +74,6 @@ _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = ( "{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}" ) -_INVOKE_PROVIDERS: Final = TypeAdapter(tuple[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, ...]).validate_python( - get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL) -) - def _response_header(headers: Mapping[str, str], name: str) -> str | None: return headers.get(name, None) @@ -562,62 +558,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): """ return False - @staticmethod - def get_bedrock_invoke_provider( - model: str, - ) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None: - """ - Helper function to get the bedrock provider from the model - - handles 4 scenarios: - 1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic` - 2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic` - 3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama` - 4. model=us.amazon.nova-pro-v1:0 -> Returns `nova` - """ - if model.startswith("invoke/"): - model = model.replace("invoke/", "", 1) - - # Special case: Check for "nova" in model name first (before "amazon") - # This handles amazon.nova-* models which would otherwise match "amazon" (Titan) - if "nova" in model.lower(): - if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): - return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova") - - _split_model: Final = model.split(".")[0] - if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): - return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model) - - # If not a known provider, check for pattern with two slashes - provider = AmazonInvokeConfig._get_provider_from_model_path(model) - if provider is not None: - return provider - - for provider in _INVOKE_PROVIDERS: - if provider in model: - return provider - return None - - @staticmethod - def _get_provider_from_model_path( - model_path: str, - ) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None: - """ - Helper function to get the provider from a model path with format: provider/model-name - - Args: - model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name') - - Returns: - Optional[str]: The provider name, or None if no valid provider found - """ - parts: Final = model_path.split("/") - if len(parts) >= 1: - provider: Final = parts[0] - if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): - return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider) - return None - def convert_messages_to_prompt(self, model: str, messages, provider, custom_prompt_dict) -> tuple[str, list | None]: # handle anthropic prompts and amazon titan prompts prompt = "" diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 87d815eb8c1..1e9b3ca2d0a 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1109,6 +1109,11 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None: return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw) +def _cost_map_row_names_a_provider(key: str) -> bool: + entry: Final = _model_cost_entry(key) + return entry is not None and entry.get("litellm_provider") is not None + + def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]: return tuple( _model_cost_entry(key) @@ -1579,7 +1584,7 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[ The deployment's ``model_info.supports_regex_lookaround`` wins, then the ``model_prices_and_context_window.json`` entry of its ``base_model``, then the entry of ``model`` itself. A model nobody flagged keeps its schema as sent, unless - every candidate is a bare id that ``bedrock_model_is_unknown`` describes. + no candidate names a model LiteLLM knows, whatever route prefix it carries. """ params: Final = litellm_params or {} deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround") @@ -1591,22 +1596,16 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[ flag: Final = next((flag for flag in flags if flag is not None), None) if flag is not None: return flag - return not all(bedrock_model_is_unknown(candidate) for candidate in candidates) + return not all(bedrock_model_is_unknown(split_bedrock_region_path(candidate)[1]) for candidate in candidates) def bedrock_model_is_unknown(model: str) -> bool: - """ - Whether ``model`` is a bare Bedrock model id that no cost-map row describes and whose vendor - has no Invoke handler, so it routes to Converse with nothing to say what the model accepts. - - An id carrying a route or provider prefix, a region path, or an ARN is never unknown. - """ model_id: Final = model.removeprefix("bedrock/") if "/" in model_id: return False base: Final = get_bedrock_base_model(model_id) regional_ids: Final = (f"{region}.{base}" for region in get_bedrock_cross_region_inference_regions()) - if any(_model_cost_entry(key) is not None for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)): + if any(_cost_map_row_names_a_provider(key) for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)): return False from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 355d01f86f8..5a6f9422362 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -1221,16 +1221,26 @@ def test_transform_request_drops_lookaround_regex_when_the_deployment_model_info assert properties == _LOOKAROUND_FREE_PROPERTIES -def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list(): +@pytest.mark.parametrize( + ("model", "litellm_params"), + [ + ("us.newvendor.model-1", None), + ("converse/us.newvendor.model-1", None), + ("us-east-1/us.newvendor.model-1", None), + ("us.newvendor.model-1", {"base_model": "bedrock/converse/us.newvendor.model-1"}), + ], +) +def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list(model, litellm_params): assert "us.newvendor.model-1" not in litellm.model_cost - assert _tool_schema_properties("us.newvendor.model-1", _ARTIFACT_DATA_ANTHROPIC_TOOL) == _LOOKAROUND_FREE_PROPERTIES + properties: Final = _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL, litellm_params) + + assert properties == _LOOKAROUND_FREE_PROPERTIES @pytest.mark.parametrize( "model", [ - "converse/us.newvendor.model-1", "arn:aws:bedrock:us-east-1:123456789012:custom-model-deployment/abc123", "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", "us.anthropic.claude-unreleased-v1:0", @@ -1254,7 +1264,16 @@ def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_whose_de assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"] -def test_a_router_deployment_of_a_model_the_cost_map_does_not_list_drops_lookaround_regex(respx_mock): +@pytest.mark.parametrize( + "deployment_params", + [ + {"model": "bedrock/us.newvendor.model-1"}, + {"model": "bedrock/converse/us.newvendor.model-1", "allowed_openai_params": ["tools"]}, + ], +) +def test_a_router_deployment_of_a_model_the_cost_map_does_not_list_drops_lookaround_regex( + respx_mock, deployment_params +): route: Final = respx_mock.post( "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.newvendor.model-1/converse" ).mock( @@ -1272,7 +1291,7 @@ def test_a_router_deployment_of_a_model_the_cost_map_does_not_list_drops_lookaro { "model_name": "unlisted", "litellm_params": { - "model": "bedrock/us.newvendor.model-1", + **deployment_params, "aws_access_key_id": "fake-access-key", "aws_secret_access_key": "fake-secret-key", "aws_region_name": "us-east-1", @@ -9375,6 +9394,26 @@ def test_model_missing_from_cost_map_sends_tools_to_converse(): assert tool_config["toolChoice"] == {"auto": {}} +def test_a_price_only_row_for_an_unlisted_model_still_sends_tools_to_converse(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, "us.newvendor.priced-1", {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6} + ) + sent: Final[list[httpx.Request]] = [] + + completion( + model="bedrock/us.newvendor.priced-1", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + tool_config: Final = json.loads(sent[0].content)["toolConfig"] + assert [tool["toolSpec"]["name"] for tool in tool_config["tools"]] == ["get_weather"] + + def test_explicit_converse_route_to_an_unlisted_model_still_drops_tools_under_drop_params(): sent: Final[list[httpx.Request]] = [] diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index f57b0e57a75..20c0e10cb81 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -219,11 +219,6 @@ def test_system_defined_inference_profile_arn_still_routes_to_converse(): ["provisioned-model/abcdefgh1234", "imported-model/abcdefgh1234", "custom-model-deployment/abcdefgh1234"], ) def test_bare_opaque_arn_with_no_invoke_provider_routes_to_converse(arn_segment: str): - """ - A bare provisioned-model, imported-model, or custom-model-deployment ARN names no Invoke provider, - so Invoke could only fail locally with "Unknown provider=None". It goes to Converse instead, and a - caller who needs Invoke for one still names the provider (``bedrock/llama/arn:...``). - """ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM model: Final = f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}"