fix(bedrock): judge lookaround support by model id on every route, ignore price-only rows

The regex check now strips route and region prefixes from the model and its
base_model before asking whether LiteLLM knows the model, so
bedrock/converse/<id> gets the same answer in a direct transform call as on the
completion path. A cost-map row counts only when it names a provider, so a
price-only register_model row leaves an unlisted id unknown, as on the proxy.
AmazonInvokeConfig now inherits the Invoke provider lookup instead of keeping
an identical copy.
This commit is contained in:
mateo-berri 2026-10-29 14:35:09 -07:00
parent d42065a37d
commit 1f2ca49601
4 changed files with 53 additions and 80 deletions

View file

@ -2,7 +2,7 @@ import copy
import json
import time
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, cast, get_args
from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
@ -74,10 +74,6 @@ _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
"{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
)
_INVOKE_PROVIDERS: Final = TypeAdapter(tuple[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, ...]).validate_python(
get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL)
)
def _response_header(headers: Mapping[str, str], name: str) -> str | None:
return headers.get(name, None)
@ -562,62 +558,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
"""
return False
@staticmethod
def get_bedrock_invoke_provider(
model: str,
) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
"""
Helper function to get the bedrock provider from the model
handles 4 scenarios:
1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama`
4. model=us.amazon.nova-pro-v1:0 -> Returns `nova`
"""
if model.startswith("invoke/"):
model = model.replace("invoke/", "", 1)
# Special case: Check for "nova" in model name first (before "amazon")
# This handles amazon.nova-* models which would otherwise match "amazon" (Titan)
if "nova" in model.lower():
if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova")
_split_model: Final = model.split(".")[0]
if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model)
# If not a known provider, check for pattern with two slashes
provider = AmazonInvokeConfig._get_provider_from_model_path(model)
if provider is not None:
return provider
for provider in _INVOKE_PROVIDERS:
if provider in model:
return provider
return None
@staticmethod
def _get_provider_from_model_path(
model_path: str,
) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
"""
Helper function to get the provider from a model path with format: provider/model-name
Args:
model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name')
Returns:
Optional[str]: The provider name, or None if no valid provider found
"""
parts: Final = model_path.split("/")
if len(parts) >= 1:
provider: Final = parts[0]
if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider)
return None
def convert_messages_to_prompt(self, model: str, messages, provider, custom_prompt_dict) -> tuple[str, list | None]:
# handle anthropic prompts and amazon titan prompts
prompt = ""

View file

@ -1109,6 +1109,11 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None:
return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw)
def _cost_map_row_names_a_provider(key: str) -> bool:
entry: Final = _model_cost_entry(key)
return entry is not None and entry.get("litellm_provider") is not None
def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]:
return tuple(
_model_cost_entry(key)
@ -1579,7 +1584,7 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
The deployment's ``model_info.supports_regex_lookaround`` wins, then the
``model_prices_and_context_window.json`` entry of its ``base_model``, then the
entry of ``model`` itself. A model nobody flagged keeps its schema as sent, unless
every candidate is a bare id that ``bedrock_model_is_unknown`` describes.
no candidate names a model LiteLLM knows, whatever route prefix it carries.
"""
params: Final = litellm_params or {}
deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround")
@ -1591,22 +1596,16 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
flag: Final = next((flag for flag in flags if flag is not None), None)
if flag is not None:
return flag
return not all(bedrock_model_is_unknown(candidate) for candidate in candidates)
return not all(bedrock_model_is_unknown(split_bedrock_region_path(candidate)[1]) for candidate in candidates)
def bedrock_model_is_unknown(model: str) -> bool:
"""
Whether ``model`` is a bare Bedrock model id that no cost-map row describes and whose vendor
has no Invoke handler, so it routes to Converse with nothing to say what the model accepts.
An id carrying a route or provider prefix, a region path, or an ARN is never unknown.
"""
model_id: Final = model.removeprefix("bedrock/")
if "/" in model_id:
return False
base: Final = get_bedrock_base_model(model_id)
regional_ids: Final = (f"{region}.{base}" for region in get_bedrock_cross_region_inference_regions())
if any(_model_cost_entry(key) is not None for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)):
if any(_cost_map_row_names_a_provider(key) for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)):
return False
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM

View file

@ -1221,16 +1221,26 @@ def test_transform_request_drops_lookaround_regex_when_the_deployment_model_info
assert properties == _LOOKAROUND_FREE_PROPERTIES
def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list():
@pytest.mark.parametrize(
("model", "litellm_params"),
[
("us.newvendor.model-1", None),
("converse/us.newvendor.model-1", None),
("us-east-1/us.newvendor.model-1", None),
("us.newvendor.model-1", {"base_model": "bedrock/converse/us.newvendor.model-1"}),
],
)
def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list(model, litellm_params):
assert "us.newvendor.model-1" not in litellm.model_cost
assert _tool_schema_properties("us.newvendor.model-1", _ARTIFACT_DATA_ANTHROPIC_TOOL) == _LOOKAROUND_FREE_PROPERTIES
properties: Final = _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL, litellm_params)
assert properties == _LOOKAROUND_FREE_PROPERTIES
@pytest.mark.parametrize(
"model",
[
"converse/us.newvendor.model-1",
"arn:aws:bedrock:us-east-1:123456789012:custom-model-deployment/abc123",
"arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123",
"us.anthropic.claude-unreleased-v1:0",
@ -1254,7 +1264,16 @@ def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_whose_de
assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"]
def test_a_router_deployment_of_a_model_the_cost_map_does_not_list_drops_lookaround_regex(respx_mock):
@pytest.mark.parametrize(
"deployment_params",
[
{"model": "bedrock/us.newvendor.model-1"},
{"model": "bedrock/converse/us.newvendor.model-1", "allowed_openai_params": ["tools"]},
],
)
def test_a_router_deployment_of_a_model_the_cost_map_does_not_list_drops_lookaround_regex(
respx_mock, deployment_params
):
route: Final = respx_mock.post(
"https://bedrock-runtime.us-east-1.amazonaws.com/model/us.newvendor.model-1/converse"
).mock(
@ -1272,7 +1291,7 @@ def test_a_router_deployment_of_a_model_the_cost_map_does_not_list_drops_lookaro
{
"model_name": "unlisted",
"litellm_params": {
"model": "bedrock/us.newvendor.model-1",
**deployment_params,
"aws_access_key_id": "fake-access-key",
"aws_secret_access_key": "fake-secret-key",
"aws_region_name": "us-east-1",
@ -9375,6 +9394,26 @@ def test_model_missing_from_cost_map_sends_tools_to_converse():
assert tool_config["toolChoice"] == {"auto": {}}
def test_a_price_only_row_for_an_unlisted_model_still_sends_tools_to_converse(monkeypatch):
monkeypatch.setitem(
litellm.model_cost, "us.newvendor.priced-1", {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}
)
sent: Final[list[httpx.Request]] = []
completion(
model="bedrock/us.newvendor.priced-1",
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
tools=[_WEATHER_TOOL],
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
aws_access_key_id="testing",
aws_secret_access_key="testing",
aws_region_name="us-west-2",
)
tool_config: Final = json.loads(sent[0].content)["toolConfig"]
assert [tool["toolSpec"]["name"] for tool in tool_config["tools"]] == ["get_weather"]
def test_explicit_converse_route_to_an_unlisted_model_still_drops_tools_under_drop_params():
sent: Final[list[httpx.Request]] = []

View file

@ -219,11 +219,6 @@ def test_system_defined_inference_profile_arn_still_routes_to_converse():
["provisioned-model/abcdefgh1234", "imported-model/abcdefgh1234", "custom-model-deployment/abcdefgh1234"],
)
def test_bare_opaque_arn_with_no_invoke_provider_routes_to_converse(arn_segment: str):
"""
A bare provisioned-model, imported-model, or custom-model-deployment ARN names no Invoke provider,
so Invoke could only fail locally with "Unknown provider=None". It goes to Converse instead, and a
caller who needs Invoke for one still names the provider (``bedrock/llama/arn:...``).
"""
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
model: Final = f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}"