mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(bedrock): judge lookaround support by model id on every route, ignore price-only rows
The regex check now strips route and region prefixes from the model and its base_model before asking whether LiteLLM knows the model, so bedrock/converse/<id> gets the same answer in a direct transform call as on the completion path. A cost-map row counts only when it names a provider, so a price-only register_model row leaves an unlisted id unknown, as on the proxy. AmazonInvokeConfig now inherits the Invoke provider lookup instead of keeping an identical copy.
This commit is contained in:
parent
d42065a37d
commit
1f2ca49601
4 changed files with 53 additions and 80 deletions
|
|
@ -2,7 +2,7 @@ import copy
|
|||
import json
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, cast, get_args
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
|
||||
|
|
@ -74,10 +74,6 @@ _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
|
|||
"{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
|
||||
)
|
||||
|
||||
_INVOKE_PROVIDERS: Final = TypeAdapter(tuple[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, ...]).validate_python(
|
||||
get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL)
|
||||
)
|
||||
|
||||
|
||||
def _response_header(headers: Mapping[str, str], name: str) -> str | None:
|
||||
return headers.get(name, None)
|
||||
|
|
@ -562,62 +558,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
"""
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_bedrock_invoke_provider(
|
||||
model: str,
|
||||
) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
|
||||
"""
|
||||
Helper function to get the bedrock provider from the model
|
||||
|
||||
handles 4 scenarios:
|
||||
1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
|
||||
2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
|
||||
3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama`
|
||||
4. model=us.amazon.nova-pro-v1:0 -> Returns `nova`
|
||||
"""
|
||||
if model.startswith("invoke/"):
|
||||
model = model.replace("invoke/", "", 1)
|
||||
|
||||
# Special case: Check for "nova" in model name first (before "amazon")
|
||||
# This handles amazon.nova-* models which would otherwise match "amazon" (Titan)
|
||||
if "nova" in model.lower():
|
||||
if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
||||
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova")
|
||||
|
||||
_split_model: Final = model.split(".")[0]
|
||||
if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
||||
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model)
|
||||
|
||||
# If not a known provider, check for pattern with two slashes
|
||||
provider = AmazonInvokeConfig._get_provider_from_model_path(model)
|
||||
if provider is not None:
|
||||
return provider
|
||||
|
||||
for provider in _INVOKE_PROVIDERS:
|
||||
if provider in model:
|
||||
return provider
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_provider_from_model_path(
|
||||
model_path: str,
|
||||
) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
|
||||
"""
|
||||
Helper function to get the provider from a model path with format: provider/model-name
|
||||
|
||||
Args:
|
||||
model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name')
|
||||
|
||||
Returns:
|
||||
Optional[str]: The provider name, or None if no valid provider found
|
||||
"""
|
||||
parts: Final = model_path.split("/")
|
||||
if len(parts) >= 1:
|
||||
provider: Final = parts[0]
|
||||
if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
||||
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider)
|
||||
return None
|
||||
|
||||
def convert_messages_to_prompt(self, model: str, messages, provider, custom_prompt_dict) -> tuple[str, list | None]:
|
||||
# handle anthropic prompts and amazon titan prompts
|
||||
prompt = ""
|
||||
|
|
|
|||
|
|
@ -1109,6 +1109,11 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None:
|
|||
return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw)
|
||||
|
||||
|
||||
def _cost_map_row_names_a_provider(key: str) -> bool:
|
||||
entry: Final = _model_cost_entry(key)
|
||||
return entry is not None and entry.get("litellm_provider") is not None
|
||||
|
||||
|
||||
def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]:
|
||||
return tuple(
|
||||
_model_cost_entry(key)
|
||||
|
|
@ -1579,7 +1584,7 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
|
|||
The deployment's ``model_info.supports_regex_lookaround`` wins, then the
|
||||
``model_prices_and_context_window.json`` entry of its ``base_model``, then the
|
||||
entry of ``model`` itself. A model nobody flagged keeps its schema as sent, unless
|
||||
every candidate is a bare id that ``bedrock_model_is_unknown`` describes.
|
||||
no candidate names a model LiteLLM knows, whatever route prefix it carries.
|
||||
"""
|
||||
params: Final = litellm_params or {}
|
||||
deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround")
|
||||
|
|
@ -1591,22 +1596,16 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
|
|||
flag: Final = next((flag for flag in flags if flag is not None), None)
|
||||
if flag is not None:
|
||||
return flag
|
||||
return not all(bedrock_model_is_unknown(candidate) for candidate in candidates)
|
||||
return not all(bedrock_model_is_unknown(split_bedrock_region_path(candidate)[1]) for candidate in candidates)
|
||||
|
||||
|
||||
def bedrock_model_is_unknown(model: str) -> bool:
|
||||
"""
|
||||
Whether ``model`` is a bare Bedrock model id that no cost-map row describes and whose vendor
|
||||
has no Invoke handler, so it routes to Converse with nothing to say what the model accepts.
|
||||
|
||||
An id carrying a route or provider prefix, a region path, or an ARN is never unknown.
|
||||
"""
|
||||
model_id: Final = model.removeprefix("bedrock/")
|
||||
if "/" in model_id:
|
||||
return False
|
||||
base: Final = get_bedrock_base_model(model_id)
|
||||
regional_ids: Final = (f"{region}.{base}" for region in get_bedrock_cross_region_inference_regions())
|
||||
if any(_model_cost_entry(key) is not None for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)):
|
||||
if any(_cost_map_row_names_a_provider(key) for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)):
|
||||
return False
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
|
||||
|
|
|
|||
|
|
@ -1221,16 +1221,26 @@ def test_transform_request_drops_lookaround_regex_when_the_deployment_model_info
|
|||
assert properties == _LOOKAROUND_FREE_PROPERTIES
|
||||
|
||||
|
||||
def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list():
|
||||
@pytest.mark.parametrize(
|
||||
("model", "litellm_params"),
|
||||
[
|
||||
("us.newvendor.model-1", None),
|
||||
("converse/us.newvendor.model-1", None),
|
||||
("us-east-1/us.newvendor.model-1", None),
|
||||
("us.newvendor.model-1", {"base_model": "bedrock/converse/us.newvendor.model-1"}),
|
||||
],
|
||||
)
|
||||
def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list(model, litellm_params):
|
||||
assert "us.newvendor.model-1" not in litellm.model_cost
|
||||
|
||||
assert _tool_schema_properties("us.newvendor.model-1", _ARTIFACT_DATA_ANTHROPIC_TOOL) == _LOOKAROUND_FREE_PROPERTIES
|
||||
properties: Final = _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL, litellm_params)
|
||||
|
||||
assert properties == _LOOKAROUND_FREE_PROPERTIES
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"converse/us.newvendor.model-1",
|
||||
"arn:aws:bedrock:us-east-1:123456789012:custom-model-deployment/abc123",
|
||||
"arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123",
|
||||
"us.anthropic.claude-unreleased-v1:0",
|
||||
|
|
@ -1254,7 +1264,16 @@ def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_whose_de
|
|||
assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"]
|
||||
|
||||
|
||||
def test_a_router_deployment_of_a_model_the_cost_map_does_not_list_drops_lookaround_regex(respx_mock):
|
||||
@pytest.mark.parametrize(
|
||||
"deployment_params",
|
||||
[
|
||||
{"model": "bedrock/us.newvendor.model-1"},
|
||||
{"model": "bedrock/converse/us.newvendor.model-1", "allowed_openai_params": ["tools"]},
|
||||
],
|
||||
)
|
||||
def test_a_router_deployment_of_a_model_the_cost_map_does_not_list_drops_lookaround_regex(
|
||||
respx_mock, deployment_params
|
||||
):
|
||||
route: Final = respx_mock.post(
|
||||
"https://bedrock-runtime.us-east-1.amazonaws.com/model/us.newvendor.model-1/converse"
|
||||
).mock(
|
||||
|
|
@ -1272,7 +1291,7 @@ def test_a_router_deployment_of_a_model_the_cost_map_does_not_list_drops_lookaro
|
|||
{
|
||||
"model_name": "unlisted",
|
||||
"litellm_params": {
|
||||
"model": "bedrock/us.newvendor.model-1",
|
||||
**deployment_params,
|
||||
"aws_access_key_id": "fake-access-key",
|
||||
"aws_secret_access_key": "fake-secret-key",
|
||||
"aws_region_name": "us-east-1",
|
||||
|
|
@ -9375,6 +9394,26 @@ def test_model_missing_from_cost_map_sends_tools_to_converse():
|
|||
assert tool_config["toolChoice"] == {"auto": {}}
|
||||
|
||||
|
||||
def test_a_price_only_row_for_an_unlisted_model_still_sends_tools_to_converse(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost, "us.newvendor.priced-1", {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}
|
||||
)
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
completion(
|
||||
model="bedrock/us.newvendor.priced-1",
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
tools=[_WEATHER_TOOL],
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
tool_config: Final = json.loads(sent[0].content)["toolConfig"]
|
||||
assert [tool["toolSpec"]["name"] for tool in tool_config["tools"]] == ["get_weather"]
|
||||
|
||||
|
||||
def test_explicit_converse_route_to_an_unlisted_model_still_drops_tools_under_drop_params():
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
|
|
|
|||
|
|
@ -219,11 +219,6 @@ def test_system_defined_inference_profile_arn_still_routes_to_converse():
|
|||
["provisioned-model/abcdefgh1234", "imported-model/abcdefgh1234", "custom-model-deployment/abcdefgh1234"],
|
||||
)
|
||||
def test_bare_opaque_arn_with_no_invoke_provider_routes_to_converse(arn_segment: str):
|
||||
"""
|
||||
A bare provisioned-model, imported-model, or custom-model-deployment ARN names no Invoke provider,
|
||||
so Invoke could only fail locally with "Unknown provider=None". It goes to Converse instead, and a
|
||||
caller who needs Invoke for one still names the provider (``bedrock/llama/arn:...``).
|
||||
"""
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
|
||||
model: Final = f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue