mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(bedrock): apply the unknown-model defaults only to bare ids LiteLLM knows nothing about
A bare Bedrock id with no cost-map row and no Invoke vendor gets Converse's open params, its reasoning_effort as sent, lookaround-free tool schemas, and a 404 for AWS's invalid-identifier reply, which keeps AWS's response headers. An explicit route like converse/, an ARN, or an id LiteLLM recognizes keeps main's params, regexes and error status, so drop_params still drops tools there
This commit is contained in:
parent
f09bf3ac35
commit
dc903ab3bc
5 changed files with 155 additions and 46 deletions
|
|
@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import (
|
|||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
|
||||
from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
|
||||
from litellm.llms.bedrock.common_utils import bedrock_model_is_unknown
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
from ..exceptions import (
|
||||
|
|
@ -901,13 +902,15 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE: Final = (
|
|||
)
|
||||
|
||||
|
||||
def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPException) -> httpx.Response:
|
||||
def _bedrock_response_with_status(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response:
|
||||
original_response: Final = getattr(original_exception, "response", None)
|
||||
if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None:
|
||||
return httpx.Response(status_code=400, request=httpx.Request(method="POST", url="https://api.openai.com/v1/"))
|
||||
if original_response.status_code == 400:
|
||||
return httpx.Response(
|
||||
status_code=status_code, request=httpx.Request(method="POST", url="https://api.openai.com/v1/")
|
||||
)
|
||||
if original_response.status_code == status_code:
|
||||
return original_response
|
||||
return httpx.Response(status_code=400, headers=original_response.headers, request=original_response.request)
|
||||
return httpx.Response(status_code=status_code, headers=original_response.headers, request=original_response.request)
|
||||
|
||||
|
||||
def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
|
||||
|
|
@ -943,7 +946,7 @@ def _map_bedrock_exception(
|
|||
message=mantle_context_window_message,
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
response=_bedrock_mantle_context_window_response(original_exception),
|
||||
response=_bedrock_response_with_status(original_exception, 400),
|
||||
)
|
||||
if (
|
||||
"too many tokens" in error_str
|
||||
|
|
@ -966,11 +969,12 @@ def _map_bedrock_exception(
|
|||
llm_provider="bedrock",
|
||||
response=getattr(original_exception, "response", None),
|
||||
)
|
||||
elif "The provided model identifier is invalid" in error_str:
|
||||
elif "The provided model identifier is invalid" in error_str and bedrock_model_is_unknown(model):
|
||||
raise NotFoundError(
|
||||
message=f"BedrockException - {error_str}",
|
||||
model=model,
|
||||
llm_provider="bedrock",
|
||||
response=_bedrock_response_with_status(original_exception, 404),
|
||||
)
|
||||
elif "Malformed input request" in error_str:
|
||||
raise BadRequestError(
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_image
|
|||
from litellm.llms.bedrock.common_utils import (
|
||||
bedrock_chat_uses_native_structured_output,
|
||||
bedrock_cleaned_output_schema,
|
||||
bedrock_model_is_unknown,
|
||||
bedrock_model_supports_regex_lookaround,
|
||||
bedrock_native_output_format,
|
||||
bedrock_native_output_is_opt_in,
|
||||
|
|
@ -116,14 +117,12 @@ from litellm.utils import (
|
|||
token_counter,
|
||||
)
|
||||
|
||||
from ..base_aws_llm import BaseAWSLLM
|
||||
from ..common_utils import (
|
||||
BedrockError,
|
||||
BedrockModelInfo,
|
||||
bedrock_cache_point_ttl_supported,
|
||||
bedrock_capability_model,
|
||||
bedrock_converse_supports_parallel_tool_use_config,
|
||||
bedrock_cost_map_has_row,
|
||||
bedrock_model_accepts_cache_points,
|
||||
bedrock_reasoning_effort_disabled,
|
||||
bedrock_rejects_stop_sequences,
|
||||
|
|
@ -528,13 +527,14 @@ class AmazonConverseConfig(BaseConfig):
|
|||
"""
|
||||
Handle the reasoning_effort parameter based on the model type.
|
||||
|
||||
- GPT-OSS and DeepSeek V3 models: passed through unchanged via additionalModelRequestFields.
|
||||
- GPT-OSS, DeepSeek V3 and unknown models (``bedrock_model_is_unknown``): passed through unchanged
|
||||
via additionalModelRequestFields.
|
||||
- OpenAI GPT-5.x and GPT-6 models: mapped to ``reasoning.effort`` via additionalModelRequestFields.
|
||||
- Nova 2 models: transformed to reasoningConfig.
|
||||
- Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on
|
||||
adaptive Claude 4.6 / 4.7).
|
||||
"""
|
||||
if "gpt-oss" in model or "deepseek" in model:
|
||||
if "gpt-oss" in model or "deepseek" in model or bedrock_model_is_unknown(model):
|
||||
optional_params["reasoning_effort"] = reasoning_effort
|
||||
elif self._is_openai_gpt_reasoning_model(model):
|
||||
reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort}
|
||||
|
|
@ -662,13 +662,6 @@ class AmazonConverseConfig(BaseConfig):
|
|||
return flag
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _has_no_cost_map_row_or_invoke_provider(model: str, base_model: str) -> bool:
|
||||
candidates: Final = (model, base_model, *(f"{prefix}{base_model}" for prefix in ("global.", "us.", "eu.")))
|
||||
if bedrock_cost_map_has_row(candidates):
|
||||
return False
|
||||
return BaseAWSLLM.get_bedrock_invoke_provider(model) is None
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[str]:
|
||||
from litellm.utils import supports_function_calling
|
||||
|
||||
|
|
@ -687,9 +680,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
"parallel_tool_calls",
|
||||
]
|
||||
|
||||
base_model: Final = BedrockModelInfo.get_base_model(model)
|
||||
|
||||
if "arn" in model or self._has_no_cost_map_row_or_invoke_provider(model=model, base_model=base_model):
|
||||
if "arn" in model or bedrock_model_is_unknown(model):
|
||||
supported_params.append("tools")
|
||||
supported_params.append("tool_choice")
|
||||
supported_params.append("thinking")
|
||||
|
|
@ -700,6 +691,9 @@ class AmazonConverseConfig(BaseConfig):
|
|||
supported_params.append("web_search_options")
|
||||
return supported_params
|
||||
|
||||
## Filter out 'cross-region' from model name
|
||||
base_model: Final = BedrockModelInfo.get_base_model(model)
|
||||
|
||||
if (
|
||||
base_model.startswith("anthropic")
|
||||
or base_model.startswith("mistral")
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import json
|
|||
import os
|
||||
import re
|
||||
import warnings
|
||||
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias
|
||||
|
|
@ -1092,10 +1092,6 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None:
|
|||
return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw)
|
||||
|
||||
|
||||
def bedrock_cost_map_has_row(keys: Iterable[str]) -> bool:
|
||||
return any(_model_cost_entry(key) is not None for key in keys)
|
||||
|
||||
|
||||
def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]:
|
||||
return tuple(
|
||||
_model_cost_entry(key)
|
||||
|
|
@ -1559,8 +1555,8 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
|
|||
|
||||
The deployment's ``model_info.supports_regex_lookaround`` wins, then the
|
||||
``model_prices_and_context_window.json`` entry of its ``base_model``, then the
|
||||
entry of ``model`` itself. A listed model nobody flagged keeps its schema as sent,
|
||||
and a model the cost map does not list drops the lookarounds.
|
||||
entry of ``model`` itself. A model nobody flagged keeps its schema as sent, unless
|
||||
every candidate is a bare id that ``bedrock_model_is_unknown`` describes.
|
||||
"""
|
||||
params: Final = litellm_params or {}
|
||||
deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround")
|
||||
|
|
@ -1572,7 +1568,26 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
|
|||
flag: Final = next((flag for flag in flags if flag is not None), None)
|
||||
if flag is not None:
|
||||
return flag
|
||||
return any(_bedrock_cost_map_lists(candidate) for candidate in candidates)
|
||||
return not all(bedrock_model_is_unknown(candidate) for candidate in candidates)
|
||||
|
||||
|
||||
def bedrock_model_is_unknown(model: str) -> bool:
|
||||
"""
|
||||
Whether ``model`` is a bare Bedrock model id that no cost-map row describes and whose vendor
|
||||
has no Invoke handler, so it routes to Converse with nothing to say what the model accepts.
|
||||
|
||||
An id carrying a route or provider prefix, a region path, or an ARN is never unknown.
|
||||
"""
|
||||
model_id: Final = model.removeprefix("bedrock/")
|
||||
if "/" in model_id:
|
||||
return False
|
||||
base: Final = get_bedrock_base_model(model_id)
|
||||
regional_ids: Final = (f"{region}.{base}" for region in get_bedrock_cross_region_inference_regions())
|
||||
if any(_model_cost_entry(key) is not None for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)):
|
||||
return False
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
|
||||
return BaseAWSLLM.get_bedrock_invoke_provider(model_id) is None
|
||||
|
||||
|
||||
_BedrockConverseModelFlag: TypeAlias = Literal[
|
||||
|
|
@ -1586,10 +1601,6 @@ def _bedrock_model_info_candidates(model: str) -> tuple[str, ...]:
|
|||
return tuple(dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base))))
|
||||
|
||||
|
||||
def _bedrock_cost_map_lists(model: str) -> bool:
|
||||
return bedrock_cost_map_has_row(_bedrock_model_info_candidates(model))
|
||||
|
||||
|
||||
def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None:
|
||||
for candidate in _bedrock_model_info_candidates(model):
|
||||
with contextlib.suppress(Exception):
|
||||
|
|
|
|||
|
|
@ -58,7 +58,6 @@ def test_openai_cyber_policy_uses_the_structured_error_code(shape: str, code: st
|
|||
assert caught.value.response.headers["x-request-id"] == "cyber-policy-request"
|
||||
|
||||
|
||||
|
||||
def _debug_info_for_api_base(configured_api_base: str) -> str:
|
||||
request: Final = httpx.Request("POST", "https://gateway.example/v1/chat/completions")
|
||||
original: Final = OpenAIError(
|
||||
|
|
@ -1468,6 +1467,24 @@ def test_bedrock_mantle_400_maps_to_bad_request():
|
|||
assert type(excinfo.value) is litellm.BadRequestError
|
||||
|
||||
|
||||
class _ErrorWithoutResponse(Exception):
|
||||
def __init__(self, message: str, status_code: int) -> None:
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
def test_bedrock_invalid_model_identifier_without_a_provider_response_still_answers_404():
|
||||
with pytest.raises(litellm.NotFoundError) as excinfo:
|
||||
exception_type(
|
||||
model="us.newvendor.typo-1",
|
||||
original_exception=_ErrorWithoutResponse("The provided model identifier is invalid.", 400),
|
||||
custom_llm_provider="bedrock",
|
||||
)
|
||||
|
||||
assert excinfo.value.response.status_code == 404
|
||||
|
||||
|
||||
def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded():
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
|
|
|
|||
|
|
@ -1214,17 +1214,29 @@ def test_transform_request_drops_lookaround_regex_when_the_deployment_model_info
|
|||
assert properties == _LOOKAROUND_FREE_PROPERTIES
|
||||
|
||||
|
||||
def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list():
|
||||
assert "us.newvendor.model-1" not in litellm.model_cost
|
||||
|
||||
assert _tool_schema_properties("us.newvendor.model-1", _ARTIFACT_DATA_ANTHROPIC_TOOL) == _LOOKAROUND_FREE_PROPERTIES
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"us.newvendor.model-1",
|
||||
"converse/us.newvendor.model-1",
|
||||
"arn:aws:bedrock:us-east-1:123456789012:custom-model-deployment/abc123",
|
||||
"arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123",
|
||||
"us.anthropic.claude-unreleased-v1:0",
|
||||
"eu.openai.gpt-unreleased",
|
||||
"apac.newvendor.listed-1",
|
||||
],
|
||||
)
|
||||
def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list(model):
|
||||
assert model not in litellm.model_cost
|
||||
def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_litellm_still_recognizes(model, monkeypatch):
|
||||
monkeypatch.setitem(litellm.model_cost, "us.newvendor.listed-1", {"litellm_provider": "bedrock_converse"})
|
||||
|
||||
assert _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL) == _LOOKAROUND_FREE_PROPERTIES
|
||||
properties = _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL)
|
||||
|
||||
assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"]
|
||||
|
||||
|
||||
def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_whose_deployment_opts_out():
|
||||
|
|
@ -5408,7 +5420,6 @@ def test_recursive_output_config_format_follows_the_profile_native_key(
|
|||
assert ("outputConfig" in request) is forwarded
|
||||
|
||||
|
||||
|
||||
def test_sonnet_4_6_json_schema_still_uses_native_output_config(monkeypatch):
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
|
|
@ -9243,31 +9254,65 @@ def test_bedrock_tools_transformation_valid_params():
|
|||
assert "test" in result[0]["toolSpec"]["inputSchema"]["json"]["required"]
|
||||
|
||||
|
||||
def test_not_found_error():
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
def _invalid_model_identifier_reply(sent: list[httpx.Request]):
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
sent.append(request)
|
||||
return httpx.Response(
|
||||
400,
|
||||
json={"message": "The provided model identifier is invalid."},
|
||||
headers={"x-amzn-ErrorType": "ValidationException"},
|
||||
headers={"x-amzn-ErrorType": "ValidationException", "x-amzn-RequestId": "req-123"},
|
||||
)
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
def test_not_found_error():
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
with pytest.raises(litellm.NotFoundError) as exc_info:
|
||||
completion(
|
||||
model="bedrock/bad_model",
|
||||
messages=[{"role": "user", "content": "What is the meaning of life"}],
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))),
|
||||
client=HTTPHandler(
|
||||
client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent)))
|
||||
),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 404
|
||||
assert exc_info.value.response.status_code == 404
|
||||
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-123"
|
||||
assert [request.url.raw_path for request in sent] == [b"/model/bad_model/converse"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, path",
|
||||
[
|
||||
("bedrock/converse/bad_model", b"/model/bad_model/converse"),
|
||||
("bedrock/anthropic.claude-typo-v1:0", b"/model/anthropic.claude-typo-v1:0/invoke"),
|
||||
],
|
||||
)
|
||||
def test_invalid_model_identifier_stays_a_bad_request_for_a_model_litellm_recognizes(model, path):
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
with pytest.raises(litellm.BadRequestError) as exc_info:
|
||||
completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "What is the meaning of life"}],
|
||||
client=HTTPHandler(
|
||||
client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent)))
|
||||
),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert [request.url.raw_path for request in sent] == [path]
|
||||
|
||||
|
||||
_WEATHER_TOOL: Final = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
|
|
@ -9287,7 +9332,9 @@ def _converse_tool_call_reply(sent: list[httpx.Request]):
|
|||
"output": {
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": [{"toolUse": {"toolUseId": "t1", "name": "get_weather", "input": {"city": "Paris"}}}],
|
||||
"content": [
|
||||
{"toolUse": {"toolUseId": "t1", "name": "get_weather", "input": {"city": "Paris"}}}
|
||||
],
|
||||
}
|
||||
},
|
||||
"stopReason": "tool_use",
|
||||
|
|
@ -9299,7 +9346,7 @@ def _converse_tool_call_reply(sent: list[httpx.Request]):
|
|||
|
||||
|
||||
def test_model_missing_from_cost_map_sends_tools_to_converse():
|
||||
model: Final = "bedrock/converse/us.newvendor.model-1"
|
||||
model: Final = "bedrock/us.newvendor.model-1"
|
||||
assert "us.newvendor.model-1" not in litellm.model_cost
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
|
|
@ -9321,6 +9368,39 @@ def test_model_missing_from_cost_map_sends_tools_to_converse():
|
|||
assert tool_config["toolChoice"] == {"auto": {}}
|
||||
|
||||
|
||||
def test_explicit_converse_route_to_an_unlisted_model_still_drops_tools_under_drop_params():
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
completion(
|
||||
model="bedrock/converse/us.newvendor.model-1",
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
tools=[_WEATHER_TOOL],
|
||||
drop_params=True,
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert "toolConfig" not in json.loads(sent[0].content)
|
||||
|
||||
|
||||
def test_unknown_model_gets_reasoning_effort_as_sent():
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
completion(
|
||||
model="bedrock/us.newvendor.model-1",
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
reasoning_effort="high",
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert json.loads(sent[0].content)["additionalModelRequestFields"] == {"reasoning_effort": "high"}
|
||||
|
||||
|
||||
def test_mapped_model_without_function_calling_still_rejects_tools():
|
||||
model: Final = "bedrock/converse/meta.llama3-8b-instruct-v1:0"
|
||||
assert "meta.llama3-8b-instruct-v1:0" in litellm.model_cost
|
||||
|
|
@ -10810,7 +10890,10 @@ def test_uses_opt_in_native_output_reads_the_base_model_of_an_application_infere
|
|||
|
||||
|
||||
_CALLER_OUTPUT_CONFIG_WITH_A_SCHEMA_OBJECT: Final = {
|
||||
"textFormat": {"type": "json_schema", "structure": {"jsonSchema": {"name": "capital", "schema": {"type": "object"}}}}
|
||||
"textFormat": {
|
||||
"type": "json_schema",
|
||||
"structure": {"jsonSchema": {"name": "capital", "schema": {"type": "object"}}},
|
||||
}
|
||||
}
|
||||
_BEDROCK_SCHEMA_TYPE_ERROR: Final = "outputConfig.textFormat.structure.jsonSchema.schema must be a string"
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue