scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-16 04:29:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 138

fix(bedrock): use base_model for Claude capability checks on application inference profile ARNs (#46569)

Browse source
This commit is contained in:
ryan-crabbe-berri 2026-10-14 20:02:57 -07:00 • committed by GitHub
parent 7b7604f0ed
commit 6c78089d34
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 483 additions and 481 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -3797,6 +3797,7 @@ class BedrockImageProcessor:
def _convert_to_bedrock_tool_call_invoke(
tool_calls: list,
model: str | None = None,
deployment_model: str | None = None,
) -> list[BedrockContentBlock]:
"""
OpenAI tool invokes:
@ -3878,6 +3879,7 @@ def _convert_to_bedrock_tool_call_invoke(
{"cache_control": tool["cache_control"]},
block_type="content_block",
model=model,
deployment_model=deployment_model,
)
if _cache_point_block is not None:
_parts_list.append(_cache_point_block)
@ -3897,6 +3899,7 @@ def _convert_to_bedrock_tool_call_invoke(
{"cache_control": tool["cache_control"]},
block_type="content_block",
model=model,
deployment_model=deployment_model,
)
if cache_point_block is not None:
_parts_list.append(cache_point_block)
@ -4537,6 +4540,7 @@ class BedrockConverseMessagesProcessor:
llm_provider: str,
user_continue_message: ChatCompletionUserMessage | None = None,
assistant_continue_message: str | ChatCompletionAssistantMessage | None = None,
deployment_model: str | None = None,
base_model: str | None = None,
) -> list[BedrockMessageBlock]:
from litellm.llms.bedrock.common_utils import bedrock_converse_accepts_redacted_thinking
@ -4599,6 +4603,7 @@ class BedrockConverseMessagesProcessor:
message_block=element,
block_type="content_block",
model=model,
deployment_model=deployment_model,
)
if _cache_point_block is not None:
_parts.append(_cache_point_block)
@ -4606,7 +4611,7 @@ class BedrockConverseMessagesProcessor:
elif message_content and isinstance(message_content, str):
_part = BedrockContentBlock(text=message_content)
_cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
message_block, block_type="content_block", model=model
message_block, block_type="content_block", model=model, deployment_model=deployment_model
)
user_content.append(_part)
if _cache_point_block is not None:
@ -4656,6 +4661,7 @@ class BedrockConverseMessagesProcessor:
{"cache_control": tool_msg_cache_control},
block_type="content_block",
model=model,
deployment_model=deployment_model,
)
if cache_point_block is not None:
tool_content.append(cache_point_block)
@ -4732,6 +4738,7 @@ class BedrockConverseMessagesProcessor:
message_block=cast(OpenAIMessageContentListBlock, element),
block_type="content_block",
model=model,
deployment_model=deployment_model,
)
if _cache_point_block is not None:
assistants_parts.append(_cache_point_block)
@ -4743,14 +4750,21 @@ class BedrockConverseMessagesProcessor:
# If content is empty/whitespace, skip it (don't add a placeholder)
# Add cache point block for assistant string content
_cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
assistant_message_block, block_type="content_block", model=model
assistant_message_block,
block_type="content_block",
model=model,
deployment_model=deployment_model,
)
if _cache_point_block is not None:
assistant_content.append(_cache_point_block)
_tool_calls = assistant_message_block.get("tool_calls", [])
if _tool_calls:
assistant_content.extend(_convert_to_bedrock_tool_call_invoke(_tool_calls, model=model))
assistant_content.extend(
_convert_to_bedrock_tool_call_invoke(
_tool_calls, model=model, deployment_model=deployment_model
)
)
msg_i += 1
@ -4931,6 +4945,7 @@ def _bedrock_converse_messages_pt(
llm_provider: str,
user_continue_message: ChatCompletionUserMessage | None = None,
assistant_continue_message: str | ChatCompletionAssistantMessage | None = None,
deployment_model: str | None = None,
base_model: str | None = None,
) -> list[BedrockMessageBlock]:
"""
@ -4999,6 +5014,7 @@ def _bedrock_converse_messages_pt(
message_block=element,
block_type="content_block",
model=model,
deployment_model=deployment_model,
)
if _cache_point_block is not None:
_parts.append(_cache_point_block)
@ -5006,7 +5022,7 @@ def _bedrock_converse_messages_pt(
elif message_content and isinstance(message_content, str):
_part = BedrockContentBlock(text=message_content)
_cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
message_block, block_type="content_block", model=model
message_block, block_type="content_block", model=model, deployment_model=deployment_model
)
user_content.append(_part)
if _cache_point_block is not None:
@ -5058,6 +5074,7 @@ def _bedrock_converse_messages_pt(
{"cache_control": tool_msg_cache_control},
block_type="content_block",
model=model,
deployment_model=deployment_model,
)
if cache_point_block is not None:
tool_content.append(cache_point_block)
@ -5137,6 +5154,7 @@ def _bedrock_converse_messages_pt(
message_block=cast(OpenAIMessageContentListBlock, element),
block_type="content_block",
model=model,
deployment_model=deployment_model,
)
if _cache_point_block is not None:
assistants_parts.append(_cache_point_block)
@ -5147,13 +5165,15 @@ def _bedrock_converse_messages_pt(
assistant_content.append(BedrockContentBlock(text=_assistant_content))
# Add cache point block for assistant string content
_cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
assistant_message_block, block_type="content_block", model=model
assistant_message_block, block_type="content_block", model=model, deployment_model=deployment_model
)
if _cache_point_block is not None:
assistant_content.append(_cache_point_block)
_tool_calls = assistant_message_block.get("tool_calls", [])
if _tool_calls:
assistant_content.extend(_convert_to_bedrock_tool_call_invoke(_tool_calls, model=model))
assistant_content.extend(
_convert_to_bedrock_tool_call_invoke(_tool_calls, model=model, deployment_model=deployment_model)
)
msg_i += 1
@ -5200,20 +5220,22 @@ def make_valid_bedrock_tool_name(input_tool_name: str) -> str:
return valid_string
def add_cache_point_tool_block(tool: dict, model: str | None = None) -> BedrockToolBlock | None:
def add_cache_point_tool_block(
tool: dict, model: str | None = None, deployment_model: str | None = None
) -> BedrockToolBlock | None:
from litellm.llms.bedrock.common_utils import (
bedrock_cache_point_ttl_supported,
bedrock_model_accepts_cache_points,
supports_bedrock_prompt_cache_ttl,
)
cache_control: Final = tool.get("cache_control", None)
if cache_control is not None and bedrock_model_accepts_cache_points(model):
if cache_control is not None and bedrock_model_accepts_cache_points(model, deployment_model):
cache_point: Final = cache_control.get("type", "ephemeral")
if cache_point == "ephemeral":
cache_point_block: Final[CachePointBlock] = {"type": "default"}
if isinstance(cache_control, dict) and "ttl" in cache_control:
ttl: Final = cache_control["ttl"]
if ttl in ["5m", "1h"] and model is not None and supports_bedrock_prompt_cache_ttl(model):
if ttl in ["5m", "1h"] and bedrock_cache_point_ttl_supported(model, deployment_model):
cache_point_block["ttl"] = ttl
return {"cachePoint": cache_point_block}
return None
@ -5243,7 +5265,9 @@ def _is_bedrock_tool_block(tool: dict) -> bool:
return isinstance(tool, dict) and ("systemTool" in tool or "toolSpec" in tool or "cachePoint" in tool)
def _bedrock_tools_pt(tools: list, model: str | None = None) -> list[BedrockToolBlock]:
def _bedrock_tools_pt(
tools: list, model: str | None = None, deployment_model: str | None = None
) -> list[BedrockToolBlock]:
"""
OpenAI tools looks like:
tools = [
@ -5363,7 +5387,7 @@ def _bedrock_tools_pt(tools: list, model: str | None = None) -> list[BedrockTool
tool_block_list.append(tool_block)
## ADD CACHE POINT TOOL BLOCK ##
cache_point_tool_block = add_cache_point_tool_block(tool, model=model)
cache_point_tool_block = add_cache_point_tool_block(tool, model=model, deployment_model=deployment_model)
if cache_point_tool_block is not None:
tool_block_list.append(cache_point_tool_block)

85
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -104,6 +104,8 @@ from litellm.utils import (
from ..common_utils import (
BedrockError,
BedrockModelInfo,
bedrock_cache_point_ttl_supported,
bedrock_capability_model,
bedrock_converse_supports_parallel_tool_use_config,
bedrock_model_accepts_cache_points,
bedrock_reasoning_effort_disabled,
@ -112,12 +114,17 @@ from ..common_utils import (
get_bedrock_tool_name,
is_bedrock_application_inference_profile_arn,
normalize_bedrock_opus_output_config_effort,
supports_bedrock_prompt_cache_ttl,
)
if TYPE_CHECKING:
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
ADAPTIVE_THINKING_ON_UNRESOLVED_ARN_WARNING: Final = (
"Sending adaptive `thinking` as a fixed thinking budget for model=%s: an application inference profile ARN "
"does not name its model, so LiteLLM cannot tell whether it supports adaptive thinking. Set `base_model` on "
"the deployment (for example `us.anthropic.claude-opus-4-8`) to send adaptive thinking as requested."
)
# Computer use tool prefixes supported by Bedrock
BEDROCK_COMPUTER_USE_TOOLS: Final = [
"computer_use_preview",
@ -757,7 +764,7 @@ class AmazonConverseConfig(BaseConfig):
return ToolChoiceValuesBlock(auto={})
def map_tool_choice_values(
self, model: str, tool_choice: str | dict, drop_params: bool
self, model: str, tool_choice: object, drop_params: bool
) -> ToolChoiceValuesBlock | None:
if tool_choice == "none":
if litellm.drop_params is True or drop_params is True:
@ -1128,6 +1135,8 @@ class AmazonConverseConfig(BaseConfig):
)
if capped is not None:
optional_params["thinking"] = capped
if is_bedrock_application_inference_profile_arn(model):
litellm.verbose_logger.warning(ADAPTIVE_THINKING_ON_UNRESOLVED_ARN_WARNING, model)
else:
litellm.verbose_logger.warning(DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING, model)
else:
@ -1341,6 +1350,7 @@ class AmazonConverseConfig(BaseConfig):
| ChatCompletionAssistantMessage,
block_type: Literal["system"],
model: str | None = None,
deployment_model: str | None = None,
) -> SystemContentBlock | None:
pass
@ -1353,6 +1363,7 @@ class AmazonConverseConfig(BaseConfig):
| ChatCompletionAssistantMessage,
block_type: Literal["content_block"],
model: str | None = None,
deployment_model: str | None = None,
) -> ContentBlock | None:
pass
@ -1364,12 +1375,13 @@ class AmazonConverseConfig(BaseConfig):
| ChatCompletionAssistantMessage,
block_type: Literal["system", "content_block"],
model: str | None = None,
deployment_model: str | None = None,
) -> SystemContentBlock | ContentBlock | None:
cache_control: Final = message_block.get("cache_control", None)
if cache_control is None or not bedrock_model_accepts_cache_points(model):
if cache_control is None or not bedrock_model_accepts_cache_points(model, deployment_model):
return None
cache_point: Final = self._build_cache_point_block(cache_control, model)
cache_point: Final = self._build_cache_point_block(cache_control, model, deployment_model)
if block_type == "system":
return SystemContentBlock(cachePoint=cache_point)
@ -1377,7 +1389,9 @@ class AmazonConverseConfig(BaseConfig):
return ContentBlock(cachePoint=cache_point)
@staticmethod
def _build_cache_point_block(control: dict | None, model: str | None = None) -> CachePointBlock:
def _build_cache_point_block(
control: Mapping[str, object] | None, model: str | None = None, deployment_model: str | None = None
) -> CachePointBlock:
"""Build a Bedrock ``cachePoint`` block from an OpenAI-style ``cache_control``/``control`` dict.
``type`` is always ``"default"`` (the only value Bedrock's Converse API
@ -1387,7 +1401,7 @@ class AmazonConverseConfig(BaseConfig):
cache_point: Final = CachePointBlock(type="default")
if isinstance(control, dict) and "ttl" in control:
ttl: Final = control["ttl"]
if ttl in ["5m", "1h"] and model is not None and supports_bedrock_prompt_cache_ttl(model):
if ttl in ("5m", "1h") and bedrock_cache_point_ttl_supported(model, deployment_model):
cache_point["ttl"] = ttl
return cache_point
@ -1510,7 +1524,7 @@ class AmazonConverseConfig(BaseConfig):
return ChatCompletionTextObject(type="text", text=text, cache_control=cache_control)
def _transform_system_message(
self, messages: list[AllMessageValues], model: str | None = None
self, messages: list[AllMessageValues], model: str | None = None, deployment_model: str | None = None
) -> tuple[list[AllMessageValues], list[SystemContentBlock]]:
leading_count: Final = next(
(i for i, m in enumerate(messages) if not is_system_message(m)),
@ -1525,14 +1539,18 @@ class AmazonConverseConfig(BaseConfig):
content = message.get("content")
if isinstance(content, str) and content:
system_content_blocks.append(SystemContentBlock(text=content))
cache_block = self.get_cache_point_block(message, block_type="system", model=model)
cache_block = self.get_cache_point_block(
message, block_type="system", model=model, deployment_model=deployment_model
)
if cache_block:
system_content_blocks.append(cache_block)
elif isinstance(content, list):
for m in content:
if m.get("type") == "text" and m.get("text"):
system_content_blocks.append(SystemContentBlock(text=m["text"]))
cache_block = self.get_cache_point_block(m, block_type="system", model=model)
cache_block = self.get_cache_point_block(
m, block_type="system", model=model, deployment_model=deployment_model
)
if cache_block:
system_content_blocks.append(cache_block)
reordered: Final = tuple(
@ -1729,6 +1747,7 @@ class AmazonConverseConfig(BaseConfig):
headers: dict | None,
additional_request_params: dict,
litellm_params: Mapping[str, object] | None = None,
deployment_model: str | None = None,
) -> tuple[list[ToolBlock], list]:
"""Process tools and collect anthropic_beta values."""
bedrock_tools: list[ToolBlock] = []
@ -1770,7 +1789,9 @@ class AmazonConverseConfig(BaseConfig):
# Process regular function tools using existing logic
bedrock_tools = _bedrock_tools_pt(
_tools_the_model_accepts(regular_tools, model, litellm_params), model=model
_tools_the_model_accepts(regular_tools, model, litellm_params),
model=model,
deployment_model=deployment_model,
)
# Add computer use tools and anthropic_beta if needed (only when computer use tools are present)
@ -1838,6 +1859,7 @@ class AmazonConverseConfig(BaseConfig):
bedrock_tools = _bedrock_tools_pt(
_tools_the_model_accepts(_TOOLS_AS_SENT.validate_python(filtered_tools), model, litellm_params),
model=model,
deployment_model=deployment_model,
)
# Append pre-formatted tools (systemTool etc.) after transformation
@ -1912,6 +1934,7 @@ class AmazonConverseConfig(BaseConfig):
headers: dict | None = None,
drop_params: bool = False,
litellm_params: Mapping[str, object] | None = None,
deployment_model: str | None = None,
) -> CommonRequestObject:
# Drop thinking param if thinking is enabled but thinking_blocks are missing
# This prevents the error: "Expected thinking or redacted_thinking, but found tool_use"
@ -1950,15 +1973,19 @@ class AmazonConverseConfig(BaseConfig):
# Process tools and collect beta values
bedrock_tools, anthropic_beta_list = self._process_tools_and_beta(
original_tools, model, headers, additional_request_params, litellm_params
original_tools, model, headers, additional_request_params, litellm_params, deployment_model
)
# Append cachePoint to tools if cache_control_injection_points has tool_config
cache_injection_points: Final = additional_request_params.pop("cache_control_injection_points", None)
if cache_injection_points and len(bedrock_tools) > 0 and bedrock_model_accepts_cache_points(model):
if (
cache_injection_points
and len(bedrock_tools) > 0
and bedrock_model_accepts_cache_points(model, deployment_model)
):
for point in cache_injection_points:
if point.get("location") == "tool_config":
cache_point = self._build_cache_point_block(point.get("control"), model)
cache_point = self._build_cache_point_block(point.get("control"), model, deployment_model)
bedrock_tools.append(ToolBlock(cachePoint=cache_point))
# Spend attribution credits the gateway only for breakpoints it placed, and
# this is the one place a tool_config point becomes one. The hook that reads
@ -2019,10 +2046,14 @@ class AmazonConverseConfig(BaseConfig):
model: str,
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
litellm_params: dict[str, object],
headers: dict | None = None,
) -> RequestObject:
messages, system_content_blocks = self._transform_system_message(messages, model=model)
base_model: Final = litellm_params.get("base_model")
capability_model: Final = bedrock_capability_model(model, base_model)
messages, system_content_blocks = self._transform_system_message(
messages, model=capability_model, deployment_model=model
)
# Convert last user message to guarded_text if guardrailConfig is present
messages = self._convert_consecutive_user_messages_to_guarded_text(
@ -2031,7 +2062,8 @@ class AmazonConverseConfig(BaseConfig):
## TRANSFORMATION ##
_data: Final[CommonRequestObject] = self._transform_request_helper(
model=model,
model=capability_model,
deployment_model=model,
system_content_blocks=system_content_blocks,
optional_params=optional_params,
messages=messages,
@ -2044,12 +2076,13 @@ class AmazonConverseConfig(BaseConfig):
place_tool_result_images(
await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
messages=messages,
model=model,
model=capability_model,
deployment_model=model,
llm_provider="bedrock_converse",
user_continue_message=litellm_params.pop("user_continue_message", None),
base_model=self._deployment_base_model(_DEPLOYMENT_PARAMS.validate_python(litellm_params)),
),
model,
capability_model,
)
)
@ -2093,10 +2126,14 @@ class AmazonConverseConfig(BaseConfig):
model: str,
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
litellm_params: dict[str, object],
headers: dict | None = None,
) -> RequestObject:
messages, system_content_blocks = self._transform_system_message(messages, model=model)
base_model: Final = litellm_params.get("base_model")
capability_model: Final = bedrock_capability_model(model, base_model)
messages, system_content_blocks = self._transform_system_message(
messages, model=capability_model, deployment_model=model
)
# Convert last user message to guarded_text if guardrailConfig is present
messages = self._convert_consecutive_user_messages_to_guarded_text(
@ -2104,7 +2141,8 @@ class AmazonConverseConfig(BaseConfig):
)
_data: Final[CommonRequestObject] = self._transform_request_helper(
model=model,
model=capability_model,
deployment_model=model,
system_content_blocks=system_content_blocks,
optional_params=optional_params,
messages=messages,
@ -2118,12 +2156,13 @@ class AmazonConverseConfig(BaseConfig):
place_tool_result_images(
_bedrock_converse_messages_pt(
messages=messages,
model=model,
model=capability_model,
deployment_model=model,
llm_provider="bedrock_converse",
user_continue_message=litellm_params.pop("user_continue_message", None),
base_model=self._deployment_base_model(_DEPLOYMENT_PARAMS.validate_python(litellm_params)),
),
model,
capability_model,
)
)

29
litellm/llms/bedrock/common_utils.py
View file

@ -812,6 +812,14 @@ def is_bedrock_application_inference_profile_arn(model: str) -> bool:
return ":application-inference-profile/" in model
def bedrock_capability_model(model: str, base_model: object) -> str:
"""The model id to look capabilities up by: the deployment's ``base_model`` for an
application inference profile ARN, since the ARN names no model, else ``model``."""
if isinstance(base_model, str) and base_model and is_bedrock_application_inference_profile_arn(model):
return strip_bedrock_routing_prefix(base_model)
return model
def strip_bedrock_routing_prefix(model: str) -> str:
"""Strip LiteLLM routing prefixes from model name."""
for prefix in ["bedrock/", "chat_completions/", "converse/", "invoke/", "openai/", "mantle/", "nova-2/", "nova/"]:
@ -1189,7 +1197,7 @@ def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool:
)
def bedrock_model_accepts_cache_points(model: str | None) -> bool:
def bedrock_model_accepts_cache_points(model: str | None, deployment_model: str | None = None) -> bool:
"""
Whether Converse ``cachePoint`` blocks may be sent to this model.
@ -1207,9 +1215,18 @@ def bedrock_model_accepts_cache_points(model: str | None) -> bool:
inference profile ARN or a model newer than the map, while only the map decides whether
a model is known: absent a map entry the model keeps emitting so existing caching setups
never silently degrade. ``litellm.utils.supports_prompt_caching`` is not reusable here:
it returns False for unmapped models, the opposite polarity.
it returns False for unmapped models, the opposite polarity. ``deployment_model`` is the
model as configured when ``model`` is its ``base_model``: its own declaration wins.
"""
return model is None or bedrock_declared_cache_point_support(model) is not False
if model is None:
return True
deployment_declared: Final = (
None if deployment_model is None else bedrock_declared_cache_point_support(deployment_model)
)
declared: Final = (
deployment_declared if deployment_declared is not None else bedrock_declared_cache_point_support(model)
)
return declared is not False
def bedrock_declared_cache_point_support(model: str) -> bool | None:
@ -1263,6 +1280,12 @@ def supports_bedrock_prompt_cache_ttl(model: str) -> bool:
)
def bedrock_cache_point_ttl_supported(model: str | None, deployment_model: str | None = None) -> bool:
return any(
supports_bedrock_prompt_cache_ttl(candidate) for candidate in (model, deployment_model) if candidate is not None
)
def is_claude_4_5_on_bedrock(model: str) -> bool:
warnings.warn(
"is_claude_4_5_on_bedrock is deprecated, use supports_bedrock_prompt_cache_ttl",

4
litellm/utils.py
View file

@ -4792,8 +4792,10 @@ def get_optional_params(
bedrock_model_info: Final[type[BedrockModelInfo]] = litellm_utils.BedrockModelInfo
bedrock_base_model: Final = bedrock_model_info.get_base_model(model)
if bedrock_route == "converse" or bedrock_route == "converse_like":
from litellm.llms.bedrock.common_utils import bedrock_capability_model
optional_params = litellm.AmazonConverseConfig().map_openai_params(
model=model,
model=bedrock_capability_model(model, base_model),
non_default_params=non_default_params,
optional_params=optional_params,
drop_params=bool(drop_params),

775
tests/unit/llms/bedrock/chat/test_converse_transformation.py
View file

File diff suppressed because it is too large Load diff

23
tests/unit/llms/bedrock/test_bedrock_common_utils.py
View file

@ -2025,3 +2025,26 @@ def test_event_decoder_reports_only_missing_aws_dependency(missing):
assert error.value.__cause__ is failure
else:
assert error.value is failure
@pytest.mark.parametrize(
("model", "base_model", "expected"),
[
(
"arn:aws:bedrock:us-east-1:000000000000:application-inference-profile/profile-id",
"bedrock/us.anthropic.claude-opus-4-8",
"us.anthropic.claude-opus-4-8",
),
(
"arn:aws:bedrock:us-east-1:000000000000:application-inference-profile/profile-id",
None,
"arn:aws:bedrock:us-east-1:000000000000:application-inference-profile/profile-id",
),
("us.anthropic.claude-opus-4-8", "anthropic.claude-sonnet-4-5", "us.anthropic.claude-opus-4-8"),
],
ids=["profile-arn-with-base-model", "profile-arn-without-base-model", "named-model-keeps-its-name"],
)
def test_bedrock_capability_model_reads_base_model_only_for_application_profile_arns(model, base_model, expected):
from litellm.llms.bedrock.common_utils import bedrock_capability_model
assert bedrock_capability_model(model, base_model) == expected