From 6c78089d34c68df9492d65fa51e463104398a0d1 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 14 Oct 2026 20:02:57 -0700 Subject: [PATCH] fix(bedrock): use base_model for Claude capability checks on application inference profile ARNs (#46569) --- .../prompt_templates/factory.py | 48 +- .../bedrock/chat/converse_transformation.py | 85 +- litellm/llms/bedrock/common_utils.py | 29 +- litellm/utils.py | 4 +- .../chat/test_converse_transformation.py | 775 ++++++++---------- .../llms/bedrock/test_bedrock_common_utils.py | 23 + 6 files changed, 483 insertions(+), 481 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 4592f6d5e45..407772ceba6 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -3797,6 +3797,7 @@ class BedrockImageProcessor: def _convert_to_bedrock_tool_call_invoke( tool_calls: list, model: str | None = None, + deployment_model: str | None = None, ) -> list[BedrockContentBlock]: """ OpenAI tool invokes: @@ -3878,6 +3879,7 @@ def _convert_to_bedrock_tool_call_invoke( {"cache_control": tool["cache_control"]}, block_type="content_block", model=model, + deployment_model=deployment_model, ) if _cache_point_block is not None: _parts_list.append(_cache_point_block) @@ -3897,6 +3899,7 @@ def _convert_to_bedrock_tool_call_invoke( {"cache_control": tool["cache_control"]}, block_type="content_block", model=model, + deployment_model=deployment_model, ) if cache_point_block is not None: _parts_list.append(cache_point_block) @@ -4537,6 +4540,7 @@ class BedrockConverseMessagesProcessor: llm_provider: str, user_continue_message: ChatCompletionUserMessage | None = None, assistant_continue_message: str | ChatCompletionAssistantMessage | None = None, + deployment_model: str | None = None, base_model: str | None = None, ) -> list[BedrockMessageBlock]: from litellm.llms.bedrock.common_utils import bedrock_converse_accepts_redacted_thinking @@ -4599,6 +4603,7 @@ class BedrockConverseMessagesProcessor: message_block=element, block_type="content_block", model=model, + deployment_model=deployment_model, ) if _cache_point_block is not None: _parts.append(_cache_point_block) @@ -4606,7 +4611,7 @@ class BedrockConverseMessagesProcessor: elif message_content and isinstance(message_content, str): _part = BedrockContentBlock(text=message_content) _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block( - message_block, block_type="content_block", model=model + message_block, block_type="content_block", model=model, deployment_model=deployment_model ) user_content.append(_part) if _cache_point_block is not None: @@ -4656,6 +4661,7 @@ class BedrockConverseMessagesProcessor: {"cache_control": tool_msg_cache_control}, block_type="content_block", model=model, + deployment_model=deployment_model, ) if cache_point_block is not None: tool_content.append(cache_point_block) @@ -4732,6 +4738,7 @@ class BedrockConverseMessagesProcessor: message_block=cast(OpenAIMessageContentListBlock, element), block_type="content_block", model=model, + deployment_model=deployment_model, ) if _cache_point_block is not None: assistants_parts.append(_cache_point_block) @@ -4743,14 +4750,21 @@ class BedrockConverseMessagesProcessor: # If content is empty/whitespace, skip it (don't add a placeholder) # Add cache point block for assistant string content _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block( - assistant_message_block, block_type="content_block", model=model + assistant_message_block, + block_type="content_block", + model=model, + deployment_model=deployment_model, ) if _cache_point_block is not None: assistant_content.append(_cache_point_block) _tool_calls = assistant_message_block.get("tool_calls", []) if _tool_calls: - assistant_content.extend(_convert_to_bedrock_tool_call_invoke(_tool_calls, model=model)) + assistant_content.extend( + _convert_to_bedrock_tool_call_invoke( + _tool_calls, model=model, deployment_model=deployment_model + ) + ) msg_i += 1 @@ -4931,6 +4945,7 @@ def _bedrock_converse_messages_pt( llm_provider: str, user_continue_message: ChatCompletionUserMessage | None = None, assistant_continue_message: str | ChatCompletionAssistantMessage | None = None, + deployment_model: str | None = None, base_model: str | None = None, ) -> list[BedrockMessageBlock]: """ @@ -4999,6 +5014,7 @@ def _bedrock_converse_messages_pt( message_block=element, block_type="content_block", model=model, + deployment_model=deployment_model, ) if _cache_point_block is not None: _parts.append(_cache_point_block) @@ -5006,7 +5022,7 @@ def _bedrock_converse_messages_pt( elif message_content and isinstance(message_content, str): _part = BedrockContentBlock(text=message_content) _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block( - message_block, block_type="content_block", model=model + message_block, block_type="content_block", model=model, deployment_model=deployment_model ) user_content.append(_part) if _cache_point_block is not None: @@ -5058,6 +5074,7 @@ def _bedrock_converse_messages_pt( {"cache_control": tool_msg_cache_control}, block_type="content_block", model=model, + deployment_model=deployment_model, ) if cache_point_block is not None: tool_content.append(cache_point_block) @@ -5137,6 +5154,7 @@ def _bedrock_converse_messages_pt( message_block=cast(OpenAIMessageContentListBlock, element), block_type="content_block", model=model, + deployment_model=deployment_model, ) if _cache_point_block is not None: assistants_parts.append(_cache_point_block) @@ -5147,13 +5165,15 @@ def _bedrock_converse_messages_pt( assistant_content.append(BedrockContentBlock(text=_assistant_content)) # Add cache point block for assistant string content _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block( - assistant_message_block, block_type="content_block", model=model + assistant_message_block, block_type="content_block", model=model, deployment_model=deployment_model ) if _cache_point_block is not None: assistant_content.append(_cache_point_block) _tool_calls = assistant_message_block.get("tool_calls", []) if _tool_calls: - assistant_content.extend(_convert_to_bedrock_tool_call_invoke(_tool_calls, model=model)) + assistant_content.extend( + _convert_to_bedrock_tool_call_invoke(_tool_calls, model=model, deployment_model=deployment_model) + ) msg_i += 1 @@ -5200,20 +5220,22 @@ def make_valid_bedrock_tool_name(input_tool_name: str) -> str: return valid_string -def add_cache_point_tool_block(tool: dict, model: str | None = None) -> BedrockToolBlock | None: +def add_cache_point_tool_block( + tool: dict, model: str | None = None, deployment_model: str | None = None +) -> BedrockToolBlock | None: from litellm.llms.bedrock.common_utils import ( + bedrock_cache_point_ttl_supported, bedrock_model_accepts_cache_points, - supports_bedrock_prompt_cache_ttl, ) cache_control: Final = tool.get("cache_control", None) - if cache_control is not None and bedrock_model_accepts_cache_points(model): + if cache_control is not None and bedrock_model_accepts_cache_points(model, deployment_model): cache_point: Final = cache_control.get("type", "ephemeral") if cache_point == "ephemeral": cache_point_block: Final[CachePointBlock] = {"type": "default"} if isinstance(cache_control, dict) and "ttl" in cache_control: ttl: Final = cache_control["ttl"] - if ttl in ["5m", "1h"] and model is not None and supports_bedrock_prompt_cache_ttl(model): + if ttl in ["5m", "1h"] and bedrock_cache_point_ttl_supported(model, deployment_model): cache_point_block["ttl"] = ttl return {"cachePoint": cache_point_block} return None @@ -5243,7 +5265,9 @@ def _is_bedrock_tool_block(tool: dict) -> bool: return isinstance(tool, dict) and ("systemTool" in tool or "toolSpec" in tool or "cachePoint" in tool) -def _bedrock_tools_pt(tools: list, model: str | None = None) -> list[BedrockToolBlock]: +def _bedrock_tools_pt( + tools: list, model: str | None = None, deployment_model: str | None = None +) -> list[BedrockToolBlock]: """ OpenAI tools looks like: tools = [ @@ -5363,7 +5387,7 @@ def _bedrock_tools_pt(tools: list, model: str | None = None) -> list[BedrockTool tool_block_list.append(tool_block) ## ADD CACHE POINT TOOL BLOCK ## - cache_point_tool_block = add_cache_point_tool_block(tool, model=model) + cache_point_tool_block = add_cache_point_tool_block(tool, model=model, deployment_model=deployment_model) if cache_point_tool_block is not None: tool_block_list.append(cache_point_tool_block) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 7cff1e7a109..b0fe0930f9d 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -104,6 +104,8 @@ from litellm.utils import ( from ..common_utils import ( BedrockError, BedrockModelInfo, + bedrock_cache_point_ttl_supported, + bedrock_capability_model, bedrock_converse_supports_parallel_tool_use_config, bedrock_model_accepts_cache_points, bedrock_reasoning_effort_disabled, @@ -112,12 +114,17 @@ from ..common_utils import ( get_bedrock_tool_name, is_bedrock_application_inference_profile_arn, normalize_bedrock_opus_output_config_effort, - supports_bedrock_prompt_cache_ttl, ) if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer +ADAPTIVE_THINKING_ON_UNRESOLVED_ARN_WARNING: Final = ( + "Sending adaptive `thinking` as a fixed thinking budget for model=%s: an application inference profile ARN " + "does not name its model, so LiteLLM cannot tell whether it supports adaptive thinking. Set `base_model` on " + "the deployment (for example `us.anthropic.claude-opus-4-8`) to send adaptive thinking as requested." +) + # Computer use tool prefixes supported by Bedrock BEDROCK_COMPUTER_USE_TOOLS: Final = [ "computer_use_preview", @@ -757,7 +764,7 @@ class AmazonConverseConfig(BaseConfig): return ToolChoiceValuesBlock(auto={}) def map_tool_choice_values( - self, model: str, tool_choice: str | dict, drop_params: bool + self, model: str, tool_choice: object, drop_params: bool ) -> ToolChoiceValuesBlock | None: if tool_choice == "none": if litellm.drop_params is True or drop_params is True: @@ -1128,6 +1135,8 @@ class AmazonConverseConfig(BaseConfig): ) if capped is not None: optional_params["thinking"] = capped + if is_bedrock_application_inference_profile_arn(model): + litellm.verbose_logger.warning(ADAPTIVE_THINKING_ON_UNRESOLVED_ARN_WARNING, model) else: litellm.verbose_logger.warning(DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING, model) else: @@ -1341,6 +1350,7 @@ class AmazonConverseConfig(BaseConfig): | ChatCompletionAssistantMessage, block_type: Literal["system"], model: str | None = None, + deployment_model: str | None = None, ) -> SystemContentBlock | None: pass @@ -1353,6 +1363,7 @@ class AmazonConverseConfig(BaseConfig): | ChatCompletionAssistantMessage, block_type: Literal["content_block"], model: str | None = None, + deployment_model: str | None = None, ) -> ContentBlock | None: pass @@ -1364,12 +1375,13 @@ class AmazonConverseConfig(BaseConfig): | ChatCompletionAssistantMessage, block_type: Literal["system", "content_block"], model: str | None = None, + deployment_model: str | None = None, ) -> SystemContentBlock | ContentBlock | None: cache_control: Final = message_block.get("cache_control", None) - if cache_control is None or not bedrock_model_accepts_cache_points(model): + if cache_control is None or not bedrock_model_accepts_cache_points(model, deployment_model): return None - cache_point: Final = self._build_cache_point_block(cache_control, model) + cache_point: Final = self._build_cache_point_block(cache_control, model, deployment_model) if block_type == "system": return SystemContentBlock(cachePoint=cache_point) @@ -1377,7 +1389,9 @@ class AmazonConverseConfig(BaseConfig): return ContentBlock(cachePoint=cache_point) @staticmethod - def _build_cache_point_block(control: dict | None, model: str | None = None) -> CachePointBlock: + def _build_cache_point_block( + control: Mapping[str, object] | None, model: str | None = None, deployment_model: str | None = None + ) -> CachePointBlock: """Build a Bedrock ``cachePoint`` block from an OpenAI-style ``cache_control``/``control`` dict. ``type`` is always ``"default"`` (the only value Bedrock's Converse API @@ -1387,7 +1401,7 @@ class AmazonConverseConfig(BaseConfig): cache_point: Final = CachePointBlock(type="default") if isinstance(control, dict) and "ttl" in control: ttl: Final = control["ttl"] - if ttl in ["5m", "1h"] and model is not None and supports_bedrock_prompt_cache_ttl(model): + if ttl in ("5m", "1h") and bedrock_cache_point_ttl_supported(model, deployment_model): cache_point["ttl"] = ttl return cache_point @@ -1510,7 +1524,7 @@ class AmazonConverseConfig(BaseConfig): return ChatCompletionTextObject(type="text", text=text, cache_control=cache_control) def _transform_system_message( - self, messages: list[AllMessageValues], model: str | None = None + self, messages: list[AllMessageValues], model: str | None = None, deployment_model: str | None = None ) -> tuple[list[AllMessageValues], list[SystemContentBlock]]: leading_count: Final = next( (i for i, m in enumerate(messages) if not is_system_message(m)), @@ -1525,14 +1539,18 @@ class AmazonConverseConfig(BaseConfig): content = message.get("content") if isinstance(content, str) and content: system_content_blocks.append(SystemContentBlock(text=content)) - cache_block = self.get_cache_point_block(message, block_type="system", model=model) + cache_block = self.get_cache_point_block( + message, block_type="system", model=model, deployment_model=deployment_model + ) if cache_block: system_content_blocks.append(cache_block) elif isinstance(content, list): for m in content: if m.get("type") == "text" and m.get("text"): system_content_blocks.append(SystemContentBlock(text=m["text"])) - cache_block = self.get_cache_point_block(m, block_type="system", model=model) + cache_block = self.get_cache_point_block( + m, block_type="system", model=model, deployment_model=deployment_model + ) if cache_block: system_content_blocks.append(cache_block) reordered: Final = tuple( @@ -1729,6 +1747,7 @@ class AmazonConverseConfig(BaseConfig): headers: dict | None, additional_request_params: dict, litellm_params: Mapping[str, object] | None = None, + deployment_model: str | None = None, ) -> tuple[list[ToolBlock], list]: """Process tools and collect anthropic_beta values.""" bedrock_tools: list[ToolBlock] = [] @@ -1770,7 +1789,9 @@ class AmazonConverseConfig(BaseConfig): # Process regular function tools using existing logic bedrock_tools = _bedrock_tools_pt( - _tools_the_model_accepts(regular_tools, model, litellm_params), model=model + _tools_the_model_accepts(regular_tools, model, litellm_params), + model=model, + deployment_model=deployment_model, ) # Add computer use tools and anthropic_beta if needed (only when computer use tools are present) @@ -1838,6 +1859,7 @@ class AmazonConverseConfig(BaseConfig): bedrock_tools = _bedrock_tools_pt( _tools_the_model_accepts(_TOOLS_AS_SENT.validate_python(filtered_tools), model, litellm_params), model=model, + deployment_model=deployment_model, ) # Append pre-formatted tools (systemTool etc.) after transformation @@ -1912,6 +1934,7 @@ class AmazonConverseConfig(BaseConfig): headers: dict | None = None, drop_params: bool = False, litellm_params: Mapping[str, object] | None = None, + deployment_model: str | None = None, ) -> CommonRequestObject: # Drop thinking param if thinking is enabled but thinking_blocks are missing # This prevents the error: "Expected thinking or redacted_thinking, but found tool_use" @@ -1950,15 +1973,19 @@ class AmazonConverseConfig(BaseConfig): # Process tools and collect beta values bedrock_tools, anthropic_beta_list = self._process_tools_and_beta( - original_tools, model, headers, additional_request_params, litellm_params + original_tools, model, headers, additional_request_params, litellm_params, deployment_model ) # Append cachePoint to tools if cache_control_injection_points has tool_config cache_injection_points: Final = additional_request_params.pop("cache_control_injection_points", None) - if cache_injection_points and len(bedrock_tools) > 0 and bedrock_model_accepts_cache_points(model): + if ( + cache_injection_points + and len(bedrock_tools) > 0 + and bedrock_model_accepts_cache_points(model, deployment_model) + ): for point in cache_injection_points: if point.get("location") == "tool_config": - cache_point = self._build_cache_point_block(point.get("control"), model) + cache_point = self._build_cache_point_block(point.get("control"), model, deployment_model) bedrock_tools.append(ToolBlock(cachePoint=cache_point)) # Spend attribution credits the gateway only for breakpoints it placed, and # this is the one place a tool_config point becomes one. The hook that reads @@ -2019,10 +2046,14 @@ class AmazonConverseConfig(BaseConfig): model: str, messages: list[AllMessageValues], optional_params: dict, - litellm_params: dict, + litellm_params: dict[str, object], headers: dict | None = None, ) -> RequestObject: - messages, system_content_blocks = self._transform_system_message(messages, model=model) + base_model: Final = litellm_params.get("base_model") + capability_model: Final = bedrock_capability_model(model, base_model) + messages, system_content_blocks = self._transform_system_message( + messages, model=capability_model, deployment_model=model + ) # Convert last user message to guarded_text if guardrailConfig is present messages = self._convert_consecutive_user_messages_to_guarded_text( @@ -2031,7 +2062,8 @@ class AmazonConverseConfig(BaseConfig): ## TRANSFORMATION ## _data: Final[CommonRequestObject] = self._transform_request_helper( - model=model, + model=capability_model, + deployment_model=model, system_content_blocks=system_content_blocks, optional_params=optional_params, messages=messages, @@ -2044,12 +2076,13 @@ class AmazonConverseConfig(BaseConfig): place_tool_result_images( await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( messages=messages, - model=model, + model=capability_model, + deployment_model=model, llm_provider="bedrock_converse", user_continue_message=litellm_params.pop("user_continue_message", None), base_model=self._deployment_base_model(_DEPLOYMENT_PARAMS.validate_python(litellm_params)), ), - model, + capability_model, ) ) @@ -2093,10 +2126,14 @@ class AmazonConverseConfig(BaseConfig): model: str, messages: list[AllMessageValues], optional_params: dict, - litellm_params: dict, + litellm_params: dict[str, object], headers: dict | None = None, ) -> RequestObject: - messages, system_content_blocks = self._transform_system_message(messages, model=model) + base_model: Final = litellm_params.get("base_model") + capability_model: Final = bedrock_capability_model(model, base_model) + messages, system_content_blocks = self._transform_system_message( + messages, model=capability_model, deployment_model=model + ) # Convert last user message to guarded_text if guardrailConfig is present messages = self._convert_consecutive_user_messages_to_guarded_text( @@ -2104,7 +2141,8 @@ class AmazonConverseConfig(BaseConfig): ) _data: Final[CommonRequestObject] = self._transform_request_helper( - model=model, + model=capability_model, + deployment_model=model, system_content_blocks=system_content_blocks, optional_params=optional_params, messages=messages, @@ -2118,12 +2156,13 @@ class AmazonConverseConfig(BaseConfig): place_tool_result_images( _bedrock_converse_messages_pt( messages=messages, - model=model, + model=capability_model, + deployment_model=model, llm_provider="bedrock_converse", user_continue_message=litellm_params.pop("user_continue_message", None), base_model=self._deployment_base_model(_DEPLOYMENT_PARAMS.validate_python(litellm_params)), ), - model, + capability_model, ) ) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 88ab9c0070d..e79865ef6bf 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -812,6 +812,14 @@ def is_bedrock_application_inference_profile_arn(model: str) -> bool: return ":application-inference-profile/" in model +def bedrock_capability_model(model: str, base_model: object) -> str: + """The model id to look capabilities up by: the deployment's ``base_model`` for an + application inference profile ARN, since the ARN names no model, else ``model``.""" + if isinstance(base_model, str) and base_model and is_bedrock_application_inference_profile_arn(model): + return strip_bedrock_routing_prefix(base_model) + return model + + def strip_bedrock_routing_prefix(model: str) -> str: """Strip LiteLLM routing prefixes from model name.""" for prefix in ["bedrock/", "chat_completions/", "converse/", "invoke/", "openai/", "mantle/", "nova-2/", "nova/"]: @@ -1189,7 +1197,7 @@ def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool: ) -def bedrock_model_accepts_cache_points(model: str | None) -> bool: +def bedrock_model_accepts_cache_points(model: str | None, deployment_model: str | None = None) -> bool: """ Whether Converse ``cachePoint`` blocks may be sent to this model. @@ -1207,9 +1215,18 @@ def bedrock_model_accepts_cache_points(model: str | None) -> bool: inference profile ARN or a model newer than the map, while only the map decides whether a model is known: absent a map entry the model keeps emitting so existing caching setups never silently degrade. ``litellm.utils.supports_prompt_caching`` is not reusable here: - it returns False for unmapped models, the opposite polarity. + it returns False for unmapped models, the opposite polarity. ``deployment_model`` is the + model as configured when ``model`` is its ``base_model``: its own declaration wins. """ - return model is None or bedrock_declared_cache_point_support(model) is not False + if model is None: + return True + deployment_declared: Final = ( + None if deployment_model is None else bedrock_declared_cache_point_support(deployment_model) + ) + declared: Final = ( + deployment_declared if deployment_declared is not None else bedrock_declared_cache_point_support(model) + ) + return declared is not False def bedrock_declared_cache_point_support(model: str) -> bool | None: @@ -1263,6 +1280,12 @@ def supports_bedrock_prompt_cache_ttl(model: str) -> bool: ) +def bedrock_cache_point_ttl_supported(model: str | None, deployment_model: str | None = None) -> bool: + return any( + supports_bedrock_prompt_cache_ttl(candidate) for candidate in (model, deployment_model) if candidate is not None + ) + + def is_claude_4_5_on_bedrock(model: str) -> bool: warnings.warn( "is_claude_4_5_on_bedrock is deprecated, use supports_bedrock_prompt_cache_ttl", diff --git a/litellm/utils.py b/litellm/utils.py index 5a0e4da573b..90e27e00e4e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4792,8 +4792,10 @@ def get_optional_params( bedrock_model_info: Final[type[BedrockModelInfo]] = litellm_utils.BedrockModelInfo bedrock_base_model: Final = bedrock_model_info.get_base_model(model) if bedrock_route == "converse" or bedrock_route == "converse_like": + from litellm.llms.bedrock.common_utils import bedrock_capability_model + optional_params = litellm.AmazonConverseConfig().map_openai_params( - model=model, + model=bedrock_capability_model(model, base_model), non_default_params=non_default_params, optional_params=optional_params, drop_params=bool(drop_params), diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index cea83e54c7b..419a25ff9e3 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -3,6 +3,7 @@ import copy import json import os import re +from collections.abc import Iterator from typing import Final from unittest.mock import MagicMock, Mock, patch @@ -40,16 +41,11 @@ def test_transform_usage(): openai_usage = config.transform_usage(usage) assert ( openai_usage.prompt_tokens - == usage["inputTokens"] - + usage["cacheReadInputTokens"] - + usage["cacheWriteInputTokens"] + == usage["inputTokens"] + usage["cacheReadInputTokens"] + usage["cacheWriteInputTokens"] ) assert openai_usage.completion_tokens == usage["outputTokens"] assert openai_usage.total_tokens == usage["totalTokens"] - assert ( - openai_usage.prompt_tokens_details.cached_tokens - == usage["cacheReadInputTokens"] - ) + assert openai_usage.prompt_tokens_details.cached_tokens == usage["cacheReadInputTokens"] assert openai_usage._cache_creation_input_tokens == usage["cacheWriteInputTokens"] assert openai_usage._cache_read_input_tokens == usage["cacheReadInputTokens"] # completion_tokens_details should always be populated @@ -97,10 +93,7 @@ def test_transform_usage_with_mismatched_cache_details_falls_back(): ) config = AmazonConverseConfig() openai_usage = config.transform_usage(usage) - assert ( - getattr(openai_usage.prompt_tokens_details, "cache_creation_token_details", None) - is None - ) + assert getattr(openai_usage.prompt_tokens_details, "cache_creation_token_details", None) is None def test_transform_usage_without_cache_details_stays_none(): @@ -116,10 +109,7 @@ def test_transform_usage_without_cache_details_stays_none(): ) config = AmazonConverseConfig() openai_usage = config.transform_usage(usage) - assert ( - getattr(openai_usage.prompt_tokens_details, "cache_creation_token_details", None) - is None - ) + assert getattr(openai_usage.prompt_tokens_details, "cache_creation_token_details", None) is None def test_bedrock_converse_1h_cache_write_billed_at_1h_rate(monkeypatch): @@ -564,6 +554,157 @@ def test_bedrock_application_profile_model_and_model_id_use_same_route(respx_moc assert route.call_count == 2 +_PROFILE_ARN: Final = "arn:aws:bedrock:us-east-1:000000000000:application-inference-profile/profile-id" +_PROFILE_ARN_CONVERSE_URL: Final = ( + "https://bedrock-runtime.us-east-1.amazonaws.com/model/" + "arn%3Aaws%3Abedrock%3Aus-east-1%3A000000000000%3Aapplication-inference-profile%2Fprofile-id/converse" +) + + +def _routed_profile_request_body( + respx_mock: respx.Router, litellm_params: dict, model_info: dict, **completion_kwargs: object +) -> dict: + route: Final = respx_mock.post(_PROFILE_ARN_CONVERSE_URL).mock( + return_value=httpx.Response( + 200, + json={ + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + }, + ) + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "claude", + "litellm_params": { + "model": f"bedrock/{_PROFILE_ARN}", + "aws_access_key_id": "AKIAPROFILEKEY", + "aws_secret_access_key": "profile-secret", + "aws_region_name": "us-east-1", + **litellm_params, + }, + "model_info": model_info, + } + ] + ) + router.completion( + model="claude", + max_tokens=16000, + drop_params=True, + client=HTTPHandler(), + **{"messages": [{"role": "user", "content": "hi"}], **completion_kwargs}, + ) + return json.loads(route.calls[0].request.content) + + +def _routed_profile_request_fields( + respx_mock: respx.Router, litellm_params: dict, model_info: dict, thinking: dict +) -> dict: + return _routed_profile_request_body(respx_mock, litellm_params, model_info, thinking=thinking).get( + "additionalModelRequestFields", {} + ) + + +@pytest.mark.parametrize( + ("litellm_params", "model_info"), + [ + ({}, {"base_model": "us.anthropic.claude-opus-4-8"}), + ({"base_model": "bedrock/us.anthropic.claude-opus-4-8"}, {}), + ], + ids=["model_info", "litellm_params"], +) +@pytest.mark.respx(assert_all_called=True) +def test_application_profile_arn_keeps_adaptive_thinking_when_base_model_is_adaptive( + litellm_params: dict, model_info: dict, respx_mock: respx.Router, local_model_cost_map: None +) -> None: + thinking: Final = {"type": "adaptive", "display": "summarized"} + + fields: Final = _routed_profile_request_fields(respx_mock, litellm_params, model_info, thinking) + + assert fields["thinking"] == thinking + + +@pytest.mark.respx(assert_all_called=True) +def test_application_profile_arn_drops_disabled_thinking_when_base_model_always_thinks( + respx_mock: respx.Router, local_model_cost_map: None +) -> None: + fields: Final = _routed_profile_request_fields( + respx_mock, {}, {"base_model": "us.anthropic.claude-fable-5"}, {"type": "disabled"} + ) + + assert "thinking" not in fields + + +@pytest.mark.respx(assert_all_called=True) +def test_application_profile_arn_without_base_model_warns_when_adaptive_thinking_becomes_a_budget( + respx_mock: respx.Router, local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level("WARNING"): + fields: Final = _routed_profile_request_fields(respx_mock, {}, {}, {"type": "adaptive"}) + + assert fields["thinking"]["type"] == "enabled" + assert any( + _PROFILE_ARN in record.getMessage() and "`base_model`" in record.getMessage() for record in caplog.records + ) + + +_CACHED_FOR_AN_HOUR: Final = {"type": "ephemeral", "ttl": "1h"} +_OPUS: Final = "us.anthropic.claude-opus-4-8" +_LLAMA: Final = "us.meta.llama3-3-70b-instruct-v1:0" + + +def _cache_points(node: object) -> Iterator[object]: + if isinstance(node, dict): + yield from ((node["cachePoint"],) if "cachePoint" in node else ()) + yield from (point for value in node.values() for point in _cache_points(value)) + elif isinstance(node, list): + yield from (point for item in node for point in _cache_points(item)) + + +@pytest.mark.parametrize( + ("model_info", "expected_cache_point"), + [ + ({"base_model": _OPUS}, {"type": "default", "ttl": "1h"}), + ({"base_model": _OPUS, "supports_prompt_cache_breakpoint": False}, None), + ({"base_model": _OPUS, "supports_prompt_cache_breakpoint": True}, {"type": "default", "ttl": "1h"}), + ({"base_model": _LLAMA}, None), + ({"base_model": _LLAMA, "supports_prompt_cache_breakpoint": True}, {"type": "default"}), + ], + ids=[ + "base_model_allows", + "deployment_disables", + "deployment_enables_keeps_ttl", + "base_model_disallows", + "deployment_enables", + ], +) +@pytest.mark.respx(assert_all_called=True) +def test_application_profile_arn_cache_points_follow_deployment_setting_then_base_model( + model_info: dict, expected_cache_point: dict | None, respx_mock: respx.Router, local_model_cost_map: None +) -> None: + body: Final = _routed_profile_request_body( + respx_mock, + {}, + model_info, + messages=[ + {"role": "system", "content": [{"type": "text", "text": "sys", "cache_control": _CACHED_FOR_AN_HOUR}]}, + {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": _CACHED_FOR_AN_HOUR}]}, + ], + tools=[ + { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object", "properties": {}}}, + "cache_control": _CACHED_FOR_AN_HOUR, + } + ], + ) + + expected: Final = [] if expected_cache_point is None else [expected_cache_point] * 3 + assert list(_cache_points(body)) == expected, body + + @pytest.mark.respx(assert_all_called=True) def test_bedrock_cross_region_inference_profile_keeps_prefix_in_converse_url(respx_mock: respx.Router) -> None: route: Final = respx_mock.post( @@ -657,14 +798,10 @@ def test_apply_tool_call_transformation_if_needed(): role="user", content=json.dumps(tool_response), ) - transformed_message, _ = config.apply_tool_call_transformation_if_needed( - message, tool_calls - ) + transformed_message, _ = config.apply_tool_call_transformation_if_needed(message, tool_calls) assert len(transformed_message.tool_calls) == 1 assert transformed_message.tool_calls[0].function.name == "test_function" - assert transformed_message.tool_calls[0].function.arguments == json.dumps( - tool_response["parameters"] - ) + assert transformed_message.tool_calls[0].function.arguments == json.dumps(tool_response["parameters"]) def test_transform_tool_call_with_cache_control(): @@ -713,12 +850,7 @@ def test_transform_tool_call_with_cache_control(): print(function_out_msg) assert function_out_msg["toolSpec"]["name"] == "get_location" assert function_out_msg["toolSpec"]["description"] == "Get the user's location" - assert ( - function_out_msg["toolSpec"]["inputSchema"]["json"]["properties"]["location"][ - "type" - ] - == "string" - ) + assert function_out_msg["toolSpec"]["inputSchema"]["json"]["properties"]["location"]["type"] == "string" transformed_cache_msg = result["toolConfig"]["tools"][1] assert "cachePoint" in transformed_cache_msg @@ -914,9 +1046,7 @@ def test_reasoning_effort_none_omits_thinking_for_anthropic_converse(model): ("bedrock/converse/us.anthropic.claude-sonnet-4-6", "minimal", "low"), ], ) -def test_reasoning_effort_sets_output_config_for_adaptive_models_converse( - model, effort, expected_effort -): +def test_reasoning_effort_sets_output_config_for_adaptive_models_converse(model, effort, expected_effort): """Adaptive Claude 4.6 / 4.7 on Bedrock Converse routes the tier via ``output_config.effort``.""" config = AmazonConverseConfig() @@ -1337,9 +1467,7 @@ def test_output_config_format_translated_to_native_output_config_converse(): assert additional.get("output_config") == {"effort": "max"} assert "format" not in additional["output_config"] assert result["outputConfig"]["textFormat"]["type"] == "json_schema" - parsed_schema = json.loads( - result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"] - ) + parsed_schema = json.loads(result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"]) assert parsed_schema == {**schema, "additionalProperties": False} @@ -1375,10 +1503,7 @@ def test_output_config_format_dropped_on_unsupported_converse_model_warns(caplog ) assert "outputConfig" not in result - assert any( - "dropping `output_config.format`" in record.getMessage() - for record in caplog.records - ) + assert any("dropping `output_config.format`" in record.getMessage() for record in caplog.records) def test_output_config_normalized_marker_does_not_leak_into_optional_params(): @@ -1414,9 +1539,7 @@ def test_output_config_normalized_marker_does_not_leak_into_optional_params(): ("bedrock/converse/us.anthropic.claude-opus-4-7", "xhigh"), ], ) -def test_output_config_effort_normalized_for_bedrock_converse_opus( - model, expected_effort -): +def test_output_config_effort_normalized_for_bedrock_converse_opus(model, expected_effort): """Bedrock Converse accepts ``xhigh`` and forwards the provider-safe effort.""" config = AmazonConverseConfig() @@ -1689,17 +1812,13 @@ def test_get_supported_openai_params_bedrock_converse(): for model in litellm.BEDROCK_CONVERSE_MODELS: print(f"Testing model: {model}") config = AmazonConverseConfig() - supported_params_without_prefix = config.get_supported_openai_params( - model=model - ) + supported_params_without_prefix = config.get_supported_openai_params(model=model) - supported_params_with_prefix = config.get_supported_openai_params( - model=f"bedrock/converse/{model}" - ) + supported_params_with_prefix = config.get_supported_openai_params(model=f"bedrock/converse/{model}") - assert set(supported_params_without_prefix) == set( - supported_params_with_prefix - ), f"Supported params mismatch for model: {model}. Without prefix: {supported_params_without_prefix}, With prefix: {supported_params_with_prefix}" + assert set(supported_params_without_prefix) == set(supported_params_with_prefix), ( + f"Supported params mismatch for model: {model}. Without prefix: {supported_params_without_prefix}, With prefix: {supported_params_with_prefix}" + ) print(f"✅ Passed for model: {model}") @@ -2191,9 +2310,7 @@ def test_transform_response_with_structured_response_calling_tool(): "output": { "message": { "content": [ - { - "text": "I'll check the current weather in San Francisco for you." - }, + {"text": "I'll check the current weather in San Francisco for you."}, { "toolUse": { "input": { @@ -2703,9 +2820,7 @@ def test_transform_request_with_function_tool(): } ] - messages = [ - {"role": "user", "content": "What's the weather like in San Francisco?"} - ] + messages = [{"role": "user", "content": "What's the weather like in San Francisco?"}] # Transform request request_data = config._transform_request( @@ -2813,22 +2928,18 @@ async def test_assistant_message_cache_control(): llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -2874,12 +2985,10 @@ async def test_assistant_message_list_content_cache_control(): llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -2932,12 +3041,10 @@ async def test_tool_message_cache_control(): llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -2951,10 +3058,7 @@ async def test_tool_message_cache_control(): # First should be tool result assert "toolResult" in tool_message_content[0] - assert ( - tool_message_content[0]["toolResult"]["content"][0]["text"] - == "Weather data: sunny, 25°C" - ) + assert tool_message_content[0]["toolResult"]["content"][0]["text"] == "Weather data: sunny, 25°C" # Second should be cachePoint assert "cachePoint" in tool_message_content[1] @@ -2996,12 +3100,10 @@ async def test_tool_message_string_content_cache_control(): llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -3012,10 +3114,7 @@ async def test_tool_message_string_content_cache_control(): # First should be tool result assert "toolResult" in tool_message_content[0] - assert ( - tool_message_content[0]["toolResult"]["content"][0]["text"] - == "Weather: sunny, 25°C" - ) + assert tool_message_content[0]["toolResult"]["content"][0]["text"] == "Weather: sunny, 25°C" # Second should be cachePoint assert "cachePoint" in tool_message_content[1] @@ -3055,9 +3154,7 @@ async def test_tool_message_search_results_maps_to_bedrock_search_result_block() "source": "Great Source of Information About Apptio", "title": "12adbd74-46bd-4a88-88b2-0048755f6eb5", "content": [ - { - "text": "Apptio is a company that makes calls to Bedrock using passthrough APIs via LiteLLM" - } + {"text": "Apptio is a company that makes calls to Bedrock using passthrough APIs via LiteLLM"} ], "citations": {"enabled": True}, } @@ -3070,12 +3167,10 @@ async def test_tool_message_search_results_maps_to_bedrock_search_result_block() model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -3084,10 +3179,7 @@ async def test_tool_message_search_results_maps_to_bedrock_search_result_block() assert tool_result["status"] == "success" assert len(tool_result["content"]) == 1 assert "searchResult" in tool_result["content"][0] - assert ( - tool_result["content"][0]["searchResult"]["title"] - == "12adbd74-46bd-4a88-88b2-0048755f6eb5" - ) + assert tool_result["content"][0]["searchResult"]["title"] == "12adbd74-46bd-4a88-88b2-0048755f6eb5" @pytest.mark.asyncio @@ -3124,12 +3216,10 @@ async def test_tool_message_empty_search_results_falls_back_to_content(): model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -3291,12 +3381,10 @@ async def test_assistant_tool_calls_cache_control(): llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -3351,12 +3439,10 @@ async def test_multiple_tool_calls_with_mixed_cache_control(): llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -3402,12 +3488,10 @@ async def test_no_cache_control_no_cache_point(): llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -3577,10 +3661,7 @@ def test_guarded_text_with_mixed_content_types(): # Third should be guardContent assert "guardContent" in content[2] - assert ( - content[2]["guardContent"]["text"]["text"] - == "This sensitive content should be guarded" - ) + assert content[2]["guardContent"]["text"]["text"] == "This sensitive content should be guarded" @pytest.mark.asyncio @@ -3675,10 +3756,7 @@ def test_guarded_text_with_tool_calls(): # Second should be guardContent assert "guardContent" in content[1] - assert ( - content[1]["guardContent"]["text"]["text"] - == "Please be careful with sensitive information" - ) + assert content[1]["guardContent"]["text"]["text"] == "Please be careful with sensitive information" # Other messages should not have guardContent for i in range(1, 3): @@ -3739,52 +3817,36 @@ def test_auto_convert_last_user_message_to_guarded_text(): } ] - optional_params = { - "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} - } + optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) # Verify the conversion assert len(converted_messages) == 1 assert converted_messages[0]["role"] == "user" assert len(converted_messages[0]["content"]) == 1 assert converted_messages[0]["content"][0]["type"] == "guarded_text" - assert ( - converted_messages[0]["content"][0]["text"] - == "What is the main topic of this legal document?" - ) + assert converted_messages[0]["content"][0]["text"] == "What is the main topic of this legal document?" def test_auto_convert_last_user_message_string_content(): """Test that last user message with string content is automatically converted to guarded_text when guardrailConfig is present.""" config = AmazonConverseConfig() - messages = [ - {"role": "user", "content": "What is the main topic of this legal document?"} - ] + messages = [{"role": "user", "content": "What is the main topic of this legal document?"}] - optional_params = { - "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} - } + optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) # Verify the conversion assert len(converted_messages) == 1 assert converted_messages[0]["role"] == "user" assert len(converted_messages[0]["content"]) == 1 assert converted_messages[0]["content"][0]["type"] == "guarded_text" - assert ( - converted_messages[0]["content"][0]["text"] - == "What is the main topic of this legal document?" - ) + assert converted_messages[0]["content"][0]["text"] == "What is the main topic of this legal document?" def test_no_conversion_when_no_guardrail_config(): @@ -3806,9 +3868,7 @@ def test_no_conversion_when_no_guardrail_config(): optional_params = {} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) # Verify no conversion happened assert converted_messages == messages @@ -3825,14 +3885,10 @@ def test_no_conversion_when_guarded_text_already_present(): } ] - optional_params = { - "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} - } + optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) # Verify no conversion happened assert converted_messages == messages @@ -3858,14 +3914,10 @@ def test_auto_convert_with_mixed_content(): } ] - optional_params = { - "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} - } + optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) # Verify the conversion assert len(converted_messages) == 1 @@ -3874,17 +3926,11 @@ def test_auto_convert_with_mixed_content(): # First element should be converted to guarded_text assert converted_messages[0]["content"][0]["type"] == "guarded_text" - assert ( - converted_messages[0]["content"][0]["text"] - == "What is the main topic of this legal document?" - ) + assert converted_messages[0]["content"][0]["text"] == "What is the main topic of this legal document?" # Second element should remain unchanged assert converted_messages[0]["content"][1]["type"] == "image_url" - assert ( - converted_messages[0]["content"][1]["image_url"]["url"] - == "https://example.com/image.jpg" - ) + assert converted_messages[0]["content"][1]["image_url"]["url"] == "https://example.com/image.jpg" def test_auto_convert_in_full_transformation(): @@ -3903,9 +3949,7 @@ def test_auto_convert_in_full_transformation(): } ] - optional_params = { - "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} - } + optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} # Test the full transformation result = config._transform_request( @@ -3925,10 +3969,7 @@ def test_auto_convert_in_full_transformation(): assert "content" in message assert len(message["content"]) == 1 assert "guardContent" in message["content"][0] - assert ( - message["content"][0]["guardContent"]["text"]["text"] - == "What is the main topic of this legal document?" - ) + assert message["content"][0]["guardContent"]["text"]["text"] == "What is the main topic of this legal document?" def test_convert_consecutive_user_messages_to_guarded_text(): @@ -3942,14 +3983,10 @@ def test_convert_consecutive_user_messages_to_guarded_text(): {"role": "user", "content": [{"type": "text", "text": "Third user message"}]}, ] - optional_params = { - "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} - } + optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) # Verify the conversion - only the last two user messages should be converted assert len(converted_messages) == 4 @@ -3984,14 +4021,10 @@ def test_convert_all_user_messages_when_all_consecutive(): {"role": "user", "content": [{"type": "text", "text": "Third user message"}]}, ] - optional_params = { - "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} - } + optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) # Verify all three user messages are converted assert len(converted_messages) == 3 @@ -4015,14 +4048,10 @@ def test_convert_consecutive_user_messages_with_string_content(): {"role": "user", "content": "Second user message"}, ] - optional_params = { - "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} - } + optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) # Verify the conversion assert len(converted_messages) == 3 @@ -4055,14 +4084,10 @@ def test_skip_consecutive_user_messages_with_existing_guarded_text(): {"role": "user", "content": [{"type": "text", "text": "Should be converted"}]}, ] - optional_params = { - "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} - } + optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) # Verify the conversion assert len(converted_messages) == 2 @@ -4647,24 +4672,22 @@ def test_drop_thinking_param_when_thinking_blocks_missing(): optional_params = {"thinking": {"type": "enabled", "budget_tokens": 1000}} # Verify the condition is detected - assert last_assistant_with_tool_calls_has_no_thinking_blocks( - messages_without_thinking_blocks - ), "Should detect missing thinking_blocks" + assert last_assistant_with_tool_calls_has_no_thinking_blocks(messages_without_thinking_blocks), ( + "Should detect missing thinking_blocks" + ) # Simulate what _transform_request_helper does if ( optional_params.get("thinking") is not None and messages_without_thinking_blocks is not None - and last_assistant_with_tool_calls_has_no_thinking_blocks( - messages_without_thinking_blocks - ) + and last_assistant_with_tool_calls_has_no_thinking_blocks(messages_without_thinking_blocks) ): if litellm.modify_params: optional_params.pop("thinking", None) - assert ( - "thinking" not in optional_params - ), "thinking param should be dropped when modify_params=True and thinking_blocks are missing" + assert "thinking" not in optional_params, ( + "thinking param should be dropped when modify_params=True and thinking_blocks are missing" + ) # Test case 2: thinking should NOT be dropped when thinking_blocks are present messages_with_thinking_blocks = [ @@ -4679,58 +4702,46 @@ def test_drop_thinking_param_when_thinking_blocks_missing(): "function": {"name": "search", "arguments": "{}"}, } ], - "thinking_blocks": [ - {"type": "thinking", "thinking": "Let me search for weather..."} - ], + "thinking_blocks": [{"type": "thinking", "thinking": "Let me search for weather..."}], }, {"role": "tool", "content": "Weather is sunny", "tool_call_id": "call_123"}, ] - optional_params_with_thinking = { - "thinking": {"type": "enabled", "budget_tokens": 1000} - } + optional_params_with_thinking = {"thinking": {"type": "enabled", "budget_tokens": 1000}} # Verify the condition is NOT detected when thinking_blocks are present - assert not last_assistant_with_tool_calls_has_no_thinking_blocks( - messages_with_thinking_blocks - ), "Should NOT detect missing thinking_blocks when they are present" + assert not last_assistant_with_tool_calls_has_no_thinking_blocks(messages_with_thinking_blocks), ( + "Should NOT detect missing thinking_blocks when they are present" + ) # Simulate what _transform_request_helper does if ( optional_params_with_thinking.get("thinking") is not None and messages_with_thinking_blocks is not None - and last_assistant_with_tool_calls_has_no_thinking_blocks( - messages_with_thinking_blocks - ) + and last_assistant_with_tool_calls_has_no_thinking_blocks(messages_with_thinking_blocks) ): if litellm.modify_params: optional_params_with_thinking.pop("thinking", None) - assert ( - "thinking" in optional_params_with_thinking - ), "thinking param should NOT be dropped when thinking_blocks are present" + assert "thinking" in optional_params_with_thinking, ( + "thinking param should NOT be dropped when thinking_blocks are present" + ) # Test case 3: thinking should NOT be dropped when modify_params=False litellm.modify_params = False - optional_params_no_modify = { - "thinking": {"type": "enabled", "budget_tokens": 1000} - } + optional_params_no_modify = {"thinking": {"type": "enabled", "budget_tokens": 1000}} # Simulate what _transform_request_helper does if ( optional_params_no_modify.get("thinking") is not None and messages_without_thinking_blocks is not None - and last_assistant_with_tool_calls_has_no_thinking_blocks( - messages_without_thinking_blocks - ) + and last_assistant_with_tool_calls_has_no_thinking_blocks(messages_without_thinking_blocks) ): if litellm.modify_params: optional_params_no_modify.pop("thinking", None) - assert ( - "thinking" in optional_params_no_modify - ), "thinking param should NOT be dropped when modify_params=False" + assert "thinking" in optional_params_no_modify, "thinking param should NOT be dropped when modify_params=False" finally: # Restore original modify_params setting @@ -4814,19 +4825,14 @@ def test_translate_response_format_native_output_config(monkeypatch): assert "fake_stream" not in result # Verify the schema content (additionalProperties: false is added by normalization) - schema_str = result["outputConfig"]["textFormat"]["structure"]["jsonSchema"][ - "schema" - ] + schema_str = result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"] parsed_schema = json.loads(schema_str) expected_schema = { **response_format["json_schema"]["schema"], "additionalProperties": False, } assert parsed_schema == expected_schema - assert ( - result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] - == "WeatherResult" - ) + assert result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] == "WeatherResult" finally: litellm.model_cost = old_cost if old_env is None: @@ -4982,9 +4988,7 @@ def test_native_structured_output_no_fake_stream(monkeypatch): assert "fake_stream" not in result # Verify the schema content - schema_str = result["outputConfig"]["textFormat"]["structure"]["jsonSchema"][ - "schema" - ] + schema_str = result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"] assert json.loads(schema_str) == { "type": "object", "properties": {"answer": {"type": "string"}}, @@ -5037,10 +5041,7 @@ def test_transform_request_with_output_config(): assert "outputConfig" in result assert result["outputConfig"]["textFormat"]["type"] == "json_schema" - assert ( - result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] - == "TestSchema" - ) + assert result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] == "TestSchema" def test_transform_request_strips_anthropic_output_config(): @@ -5161,10 +5162,7 @@ def test_transform_response_native_structured_output(): ) # Content should be the JSON text directly - assert ( - result.choices[0].message.content - == '{"temp": 62, "description": "Mild and foggy"}' - ) + assert result.choices[0].message.content == '{"temp": 62, "description": "Mild and foggy"}' # Should NOT have tool_calls assert result.choices[0].message.tool_calls is None assert result.choices[0].finish_reason == "stop" @@ -5277,10 +5275,7 @@ def test_add_additional_properties_definitions(): # definitions object assert result["definitions"]["Item"]["additionalProperties"] is False # Nested object inside definitions - assert ( - result["definitions"]["Item"]["properties"]["details"]["additionalProperties"] - is False - ) + assert result["definitions"]["Item"]["properties"]["details"]["additionalProperties"] is False def test_json_object_no_schema_skips_tool_injection(monkeypatch): @@ -5337,9 +5332,7 @@ def test_output_config_applies_additional_properties(): output_config = AmazonConverseConfig._create_output_config_for_response_format( json_schema=schema, name="test_schema" ) - parsed = json.loads( - output_config["textFormat"]["structure"]["jsonSchema"]["schema"] - ) + parsed = json.loads(output_config["textFormat"]["structure"]["jsonSchema"]["schema"]) assert parsed["additionalProperties"] is False assert parsed["properties"]["nested"]["additionalProperties"] is False @@ -5388,12 +5381,7 @@ def test_parallel_tool_calls_newer_model_adds_disable_flag(): assert "additionalModelRequestFields" in request_data assert "tool_choice" in request_data["additionalModelRequestFields"] - assert ( - request_data["additionalModelRequestFields"]["tool_choice"][ - "disable_parallel_tool_use" - ] - is True - ) + assert request_data["additionalModelRequestFields"]["tool_choice"]["disable_parallel_tool_use"] is True assert "parallel_tool_calls" not in request_data["additionalModelRequestFields"] @@ -5425,12 +5413,7 @@ def test_parallel_tool_calls_flag_decoupled_from_ttl_pricing(monkeypatch): headers={}, ) - assert ( - request_data["additionalModelRequestFields"]["tool_choice"][ - "disable_parallel_tool_use" - ] - is True - ) + assert request_data["additionalModelRequestFields"]["tool_choice"]["disable_parallel_tool_use"] is True def test_parallel_tool_calls_older_model_drops_disable_flag(): @@ -5577,9 +5560,7 @@ def test_parallel_tool_use_merge_preserves_user_tool_choice_type(): class TestBedrockMinThinkingBudgetTokens: """Test that thinking.budget_tokens is clamped to the Bedrock minimum (1024).""" - def _map_params( - self, thinking_value, model="anthropic.claude-sonnet-4-5-20250929-v1:0" - ): + def _map_params(self, thinking_value, model="anthropic.claude-sonnet-4-5-20250929-v1:0"): """Helper to call map_openai_params with the given thinking value.""" config = AmazonConverseConfig() non_default_params = {"thinking": thinking_value} @@ -5806,9 +5787,7 @@ def test_streaming_filters_json_tool_call_with_real_tools(): # Chunk 2: json_tool_call delta — should become text, not tool_use json_delta = ContentBlockDeltaEvent(toolUse={"input": '{"temp": 62}'}) - text_2, tool_use_2, _, _, _ = decoder._handle_converse_delta_event( - json_delta, index=0 - ) + text_2, tool_use_2, _, _, _ = decoder._handle_converse_delta_event(json_delta, index=0) assert text_2 == '{"temp": 62}' assert tool_use_2 is None @@ -5832,9 +5811,7 @@ def test_streaming_filters_json_tool_call_with_real_tools(): # Chunk 5: real tool delta real_delta = ContentBlockDeltaEvent(toolUse={"input": '{"location": "SF"}'}) - text_5, tool_use_5, _, _, _ = decoder._handle_converse_delta_event( - real_delta, index=1 - ) + text_5, tool_use_5, _, _, _ = decoder._handle_converse_delta_event(real_delta, index=1) assert text_5 == "" assert tool_use_5 is not None assert tool_use_5["function"]["arguments"] == '{"location": "SF"}' @@ -5867,9 +5844,7 @@ def test_streaming_without_json_mode_passes_all_tools(): # json_tool_call delta — should be a tool_use, not text json_delta = ContentBlockDeltaEvent(toolUse={"input": '{"data": 1}'}) - text, tool_use_delta, _, _, _ = decoder._handle_converse_delta_event( - json_delta, index=0 - ) + text, tool_use_delta, _, _, _ = decoder._handle_converse_delta_event(json_delta, index=0) assert text == "" assert tool_use_delta is not None assert tool_use_delta["function"]["arguments"] == '{"data": 1}' @@ -6222,8 +6197,6 @@ def test_tool_config_cachepoint_not_placed_or_credited_for_model_without_prompt_ assert "litellm_gateway_injected_cache" not in bucket - - def test_transform_response_finish_reason_stop_when_json_mode_filters_all_tools(): """ When json_mode is True and _filter_json_mode_tools strips all synthetic @@ -6479,11 +6452,7 @@ def test_transform_response_citation_null_source_title_become_empty_strings(): "content": [ { "citationsContent": { - "content": [ - { - "text": "Apptio is a company that makes calls to Bedrock" - } - ], + "content": [{"text": "Apptio is a company that makes calls to Bedrock"}], "citations": [ { "location": { @@ -6618,15 +6587,11 @@ def test_transform_response_citations_offset_tracks_text_only_blocks(): message = result.choices[0].message expected_start = len(leading_text) assert message.content == leading_text + cited_text - assert ( - message.content[expected_start : expected_start + len(cited_text)] == cited_text - ) + assert message.content[expected_start : expected_start + len(cited_text)] == cited_text assert message.annotations is not None assert len(message.annotations) == 1 assert message.annotations[0]["url_citation"]["start_index"] == expected_start - assert message.annotations[0]["url_citation"]["end_index"] == expected_start + len( - cited_text - ) + assert message.annotations[0]["url_citation"]["end_index"] == expected_start + len(cited_text) def test_transform_response_stitches_citations_for_whitespace_punctuation_text(): @@ -6736,9 +6701,7 @@ def test_bedrock_tool_message_openai_file_pdf_becomes_document(): }, ] - translated_msg = _bedrock_converse_messages_pt( - messages=messages, model="", llm_provider="" - ) + translated_msg = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="") tool_result = translated_msg[-1]["content"][-1]["toolResult"] assert tool_result["toolUseId"] == "tooluse_pdf_1" @@ -6780,9 +6743,7 @@ def test_bedrock_tool_message_image_url_pdf_data_uri_becomes_document(): }, ] - translated_msg = _bedrock_converse_messages_pt( - messages=messages, model="", llm_provider="" - ) + translated_msg = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="") tool_result = translated_msg[-1]["content"][-1]["toolResult"] assert tool_result["toolUseId"] == "tooluse_pdf_img_1" @@ -6839,9 +6800,7 @@ def test_bedrock_tool_message_file_id_http_url_becomes_document(): "process_image_sync", return_value=fake_document_block, ) as mock_proc: - translated_msg = _bedrock_converse_messages_pt( - messages=messages, model="", llm_provider="" - ) + translated_msg = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="") mock_proc.assert_called_once() assert mock_proc.call_args.kwargs["image_url"] == pdf_url @@ -6912,9 +6871,7 @@ def test_bedrock_tool_message_image_url_png_still_becomes_image(): }, ] - translated_msg = _bedrock_converse_messages_pt( - messages=messages, model="", llm_provider="" - ) + translated_msg = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="") tool_result = translated_msg[-1]["content"][-1]["toolResult"] assert len(tool_result["content"]) == 1 @@ -6939,9 +6896,7 @@ def _png_tool_messages(text: str | None = None) -> list[AllMessageValues]: type="image_url", image_url=ChatCompletionImageUrlObject(url=f"data:image/png;base64,{png_b64}"), ) - parts: Final = ( - (ChatCompletionTextObject(type="text", text=text), image) if text is not None else (image,) - ) + parts: Final = (ChatCompletionTextObject(type="text", text=text), image) if text is not None else (image,) user: Final = ChatCompletionUserMessage(role="user", content="Describe the attached image.") tool: Final = ChatCompletionToolMessage(role="tool", tool_call_id="tooluse_png_gpt", content=list(parts)) return [user, tool] @@ -7184,12 +7139,10 @@ async def test_grounding_source_and_query_rendered_as_text(): model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -7233,9 +7186,7 @@ def test_neutralize_orphaned_tool_blocks_rewrites_when_no_tools(): (#24158, #27138).""" messages = _orphaned_tool_history_messages() - result = AmazonConverseConfig._neutralize_orphaned_tool_blocks( - messages, optional_params={} - ) + result = AmazonConverseConfig._neutralize_orphaned_tool_blocks(messages, optional_params={}) serialized = json.dumps(result) assert "tool_calls" not in serialized @@ -7256,9 +7207,7 @@ def test_neutralize_orphaned_tool_blocks_rewrites_when_tools_empty(tools_value): structured tool blocks with no toolConfig.""" messages = _orphaned_tool_history_messages() - result = AmazonConverseConfig._neutralize_orphaned_tool_blocks( - messages, optional_params={"tools": tools_value} - ) + result = AmazonConverseConfig._neutralize_orphaned_tool_blocks(messages, optional_params={"tools": tools_value}) serialized = json.dumps(result) assert "tool_calls" not in serialized @@ -7274,9 +7223,7 @@ def test_neutralize_orphaned_tool_blocks_rewrites_tool_result_only_history(): {"role": "tool", "tool_call_id": "call_xyz", "content": "lookup result"}, ] - result = AmazonConverseConfig._neutralize_orphaned_tool_blocks( - messages, optional_params={} - ) + result = AmazonConverseConfig._neutralize_orphaned_tool_blocks(messages, optional_params={}) assert not any(m.get("role") in ("tool", "function") for m in result) serialized = json.dumps(result) @@ -7313,13 +7260,9 @@ def test_neutralize_orphaned_tool_blocks_non_text_result_marked_not_empty(): }, ] - result = AmazonConverseConfig._neutralize_orphaned_tool_blocks( - messages, optional_params={} - ) + result = AmazonConverseConfig._neutralize_orphaned_tool_blocks(messages, optional_params={}) - rewritten = next( - m for m in result if m.get("role") == "user" and m is not messages[0] - ) + rewritten = next(m for m in result if m.get("role") == "user" and m is not messages[0]) text = rewritten["content"] assert text.strip() # never empty assert "non-text tool result omitted" in text @@ -7342,9 +7285,7 @@ def test_neutralize_orphaned_tool_blocks_noop_when_no_tool_history(): """Plain conversation with no tool blocks is returned unchanged.""" messages = [{"role": "user", "content": "hi"}] - result = AmazonConverseConfig._neutralize_orphaned_tool_blocks( - messages, optional_params={} - ) + result = AmazonConverseConfig._neutralize_orphaned_tool_blocks(messages, optional_params={}) assert result is messages @@ -7355,14 +7296,9 @@ def test_neutralize_orphaned_tool_blocks_logs_warning(caplog): messages = _orphaned_tool_history_messages() with caplog.at_level("WARNING"): - AmazonConverseConfig._neutralize_orphaned_tool_blocks( - messages, optional_params={} - ) + AmazonConverseConfig._neutralize_orphaned_tool_blocks(messages, optional_params={}) - assert any( - "neutralizing orphaned tool blocks" in record.getMessage() - for record in caplog.records - ) + assert any("neutralizing orphaned tool blocks" in record.getMessage() for record in caplog.records) def _assert_no_structured_tool_blocks(result): @@ -7480,9 +7416,7 @@ def test_transform_request_neutralized_tool_output_is_guarded(monkeypatch): }, {"role": "tool", "tool_call_id": "c1", "content": "secret tool output"}, ], - optional_params={ - "guardrailConfig": {"guardrailIdentifier": "gid", "guardrailVersion": "1"} - }, + optional_params={"guardrailConfig": {"guardrailIdentifier": "gid", "guardrailVersion": "1"}}, litellm_params={}, headers={}, ) @@ -7522,23 +7456,19 @@ def test_transform_request_neutralized_tool_output_guarded_mid_history(monkeypat {"role": "assistant", "content": "Here is the summary."}, {"role": "user", "content": "thanks"}, ], - optional_params={ - "guardrailConfig": {"guardrailIdentifier": "gid", "guardrailVersion": "1"} - }, + optional_params={"guardrailConfig": {"guardrailIdentifier": "gid", "guardrailVersion": "1"}}, litellm_params={}, headers={}, ) _assert_no_structured_tool_blocks(result) blocks = [block for message in result["messages"] for block in message["content"]] - guarded_texts = [ - block["guardContent"]["text"]["text"] for block in blocks if "guardContent" in block - ] + guarded_texts = [block["guardContent"]["text"]["text"] for block in blocks if "guardContent" in block] plain_texts = [block["text"] for block in blocks if "text" in block and "guardContent" not in block] assert any("malware" in text for text in guarded_texts), "mid-history tool output must be guarded" - assert not any( - "malware" in text for text in plain_texts - ), "mid-history tool output must not reach the model as unguarded text" + assert not any("malware" in text for text in plain_texts), ( + "mid-history tool output must not reach the model as unguarded text" + ) @pytest.mark.asyncio @@ -7674,10 +7604,7 @@ def _agentic_messages_with_ttl(ttl_target: str): def _collect_cache_points(result): return [ - block["cachePoint"] - for message in result - for block in message.get("content") or [] - if "cachePoint" in block + block["cachePoint"] for message in result for block in message.get("content") or [] if "cachePoint" in block ] @@ -7703,12 +7630,10 @@ async def test_message_level_cache_control_honors_ttl_for_supported_model( model="global.anthropic.claude-opus-4-7", llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="global.anthropic.claude-opus-4-7", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="global.anthropic.claude-opus-4-7", + llm_provider="bedrock_converse", ) assert result == async_result @@ -8020,9 +7945,7 @@ def test_update_optional_params_with_thinking_tokens_bool_thinking_does_not_cras ("us.anthropic.claude-opus-4-8", False), ], ) -def test_disabled_thinking_omitted_for_always_on_models_converse( - local_model_cost_map, model, expected_dropped -): +def test_disabled_thinking_omitted_for_always_on_models_converse(local_model_cost_map, model, expected_dropped): """Bedrock Converse: ``thinking={"type": "disabled"}`` is omitted for always-on-thinking models and forwarded verbatim for models that accept it.""" config = AmazonConverseConfig() @@ -8041,6 +7964,7 @@ def test_disabled_thinking_omitted_for_always_on_models_converse( else: assert additional.get("thinking") == {"type": "disabled"} + @pytest.mark.parametrize( "model", ["anthropic.claude-fable-5-1", "us.anthropic.claude-fable-5-1"], @@ -8049,14 +7973,10 @@ def test_disabled_thinking_omitted_for_always_on_models_converse( "tool_choice", ["required", {"type": "function", "function": {"name": "get_weather"}}], ) -def test_forced_tool_choice_downgraded_to_auto_on_fable_5_1_converse( - local_model_cost_map, model, tool_choice -): +def test_forced_tool_choice_downgraded_to_auto_on_fable_5_1_converse(local_model_cost_map, model, tool_choice): config = AmazonConverseConfig() - result = config.map_tool_choice_values( - model=model, tool_choice=tool_choice, drop_params=True - ) + result = config.map_tool_choice_values(model=model, tool_choice=tool_choice, drop_params=True) assert result == {"auto": {}} @@ -8065,16 +7985,12 @@ def test_forced_tool_choice_downgraded_to_auto_on_fable_5_1_converse( "tool_choice", ["required", {"type": "function", "function": {"name": "get_weather"}}], ) -def test_forced_tool_choice_raises_clean_error_on_fable_5_1_converse( - local_model_cost_map, tool_choice, monkeypatch -): +def test_forced_tool_choice_raises_clean_error_on_fable_5_1_converse(local_model_cost_map, tool_choice, monkeypatch): monkeypatch.setattr(litellm, "drop_params", False) config = AmazonConverseConfig() with pytest.raises(litellm.utils.UnsupportedParamsError, match="forced tool use"): - config.map_tool_choice_values( - model="anthropic.claude-fable-5-1", tool_choice=tool_choice, drop_params=False - ) + config.map_tool_choice_values(model="anthropic.claude-fable-5-1", tool_choice=tool_choice, drop_params=False) @pytest.mark.parametrize("tool_choice", ["auto", "none"]) @@ -8092,9 +8008,7 @@ def test_unforced_tool_choice_unaffected_on_fable_5_1_converse(local_model_cost_ "model", ["anthropic.claude-fable-5-1", "us.anthropic.claude-fable-5-1"], ) -def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_converse( - local_model_cost_map, model -): +def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_converse(local_model_cost_map, model): """Regression: Bedrock rejects both ``outputConfig`` structured output and forced tool_choice for Fable 5.1, so response_format must map to a tool without a forced tool_choice.""" @@ -8121,15 +8035,11 @@ def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_conve assert result.get("json_mode") is True -def test_forced_tool_choice_forwarded_on_converse_models_that_support_it( - local_model_cost_map, monkeypatch -): +def test_forced_tool_choice_forwarded_on_converse_models_that_support_it(local_model_cost_map, monkeypatch): monkeypatch.setattr(litellm, "drop_params", False) config = AmazonConverseConfig() - result = config.map_tool_choice_values( - model="anthropic.claude-fable-5", tool_choice="required", drop_params=False - ) + result = config.map_tool_choice_values(model="anthropic.claude-fable-5", tool_choice="required", drop_params=False) assert result == {"any": {}} @@ -8470,7 +8380,11 @@ def test_system_entry_without_content_key_transforms_like_an_empty_one(): assert config._transform_system_message(leading_without_key) == config._transform_system_message(leading_empty) assert config._transform_system_message(leading_without_key) == ([{"role": "user", "content": "hi"}], []) mid_without_key = [{"role": "user", "content": "hi"}, {"role": "system"}, {"role": "user", "content": "done"}] - mid_empty = [{"role": "user", "content": "hi"}, {"role": "system", "content": ""}, {"role": "user", "content": "done"}] + mid_empty = [ + {"role": "user", "content": "hi"}, + {"role": "system", "content": ""}, + {"role": "user", "content": "done"}, + ] assert config._transform_system_message(mid_without_key) == config._transform_system_message(mid_empty) @@ -8488,7 +8402,11 @@ def _preserved_thinking_turns(reminder_after_user: bool) -> tuple[list[dict], li second_question = {"role": "user", "content": "Second question"} second_turn = [second_question, reminder] if reminder_after_user else [reminder, second_question] turn_n_plus_one = [*turn_n, _thinking_reply("First answer"), *second_turn] - turn_n_plus_two = [*turn_n_plus_one, _thinking_reply("Second answer"), {"role": "user", "content": "Third question"}] + turn_n_plus_two = [ + *turn_n_plus_one, + _thinking_reply("Second answer"), + {"role": "user", "content": "Third question"}, + ] return turn_n, turn_n_plus_one, turn_n_plus_two @@ -8620,16 +8538,6 @@ def test_supports_sampling_params_prefixed_and_anthropic_fallback(monkeypatch: p assert AmazonConverseConfig._supports_sampling_params("anthropic.claude-custom-unregistered") is True - - - - - - - - - - @pytest.mark.usefixtures("fake_provider_credentials") def test_bedrock_tools_pt_valid_names(): """ @@ -8882,24 +8790,18 @@ def test_bedrock_converse_translation_tool_message(): }, ] - translated_msg = _bedrock_converse_messages_pt( - messages=messages, model="", llm_provider="" - ) + translated_msg = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="") print(translated_msg) assert translated_msg == [ { "role": "user", "content": [ - { - "text": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses" - }, + {"text": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses"}, { "toolResult": { "content": [ - { - "text": '{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}' - } + {"text": '{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}'} ], "toolUseId": "tooluse_DnqEmD5qR6y2-aJ-Xd05xw", } @@ -9364,10 +9266,7 @@ def test_bedrock_empty_content_handling(messages, continue_message_index): # Verify assistant message with default text was inserted assert formatted_messages[0]["role"] == "user" assert formatted_messages[1]["role"] == "assistant" - assert ( - formatted_messages[continue_message_index]["content"][0]["text"] - == "Please continue." - ) + assert formatted_messages[continue_message_index]["content"][0]["text"] == "Please continue." def test_bedrock_custom_continue_message(): @@ -9416,9 +9315,7 @@ def test_bedrock_no_default_message(): ) # Verify empty message is replaced with placeholder and valid message remains - assistant_messages = [ - msg for msg in formatted_messages if msg["role"] == "assistant" - ] + assistant_messages = [msg for msg in formatted_messages if msg["role"] == "assistant"] assert len(assistant_messages) == 1 assert assistant_messages[0]["content"][0]["text"] == "Valid response" @@ -9528,9 +9425,7 @@ def test_bedrock_top_k_param(model, expected_params): mock_response = Mock() if "mistral" in model: - mock_response.text = json.dumps( - {"outputs": [{"text": "Here's a joke...", "stop_reason": "stop"}]} - ) + mock_response.text = json.dumps({"outputs": [{"text": "Here's a joke...", "stop_reason": "stop"}]}) else: mock_response.text = json.dumps( { @@ -9978,9 +9873,7 @@ def test_cross_region_bedrock_model_cost_falls_back_to_base_model(monkeypatch: p prompt="synthetic prompt", completion="synthetic completion", ) - base_info: Final = litellm.get_model_info( - model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0" - ) + base_info: Final = litellm.get_model_info(model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0") assert info["input_cost_per_token"] == base_info["input_cost_per_token"] assert cost == pytest.approx(base_cost) @@ -10090,9 +9983,7 @@ def test_bedrock_image_url_sync_client_fetches_and_inlines_image(respx_mock: res respx_mock.get("https://1.1.1.1/image.png").mock( return_value=httpx.Response(200, content=image_bytes, headers={"content-type": "image/png"}) ) - bedrock_route: Final = respx_mock.post( - re.compile(r"https://[^/]+/.*/converse(?:\?.*)?") - ).mock( + bedrock_route: Final = respx_mock.post(re.compile(r"https://[^/]+/.*/converse(?:\?.*)?")).mock( return_value=httpx.Response( 200, json={ @@ -10123,9 +10014,9 @@ def test_bedrock_image_url_sync_client_fetches_and_inlines_image(respx_mock: res assert response.choices[0].message.content == "a synthetic image" assert request_body["messages"][0]["content"][1]["image"]["format"] == "png" - assert request_body["messages"][0]["content"][1]["image"]["source"]["bytes"] == base64.b64encode( - image_bytes - ).decode() + assert ( + request_body["messages"][0]["content"][1]["image"]["source"]["bytes"] == base64.b64encode(image_bytes).decode() + ) async def _transform_converse_request(sync_mode: bool, messages: list[AllMessageValues]) -> RequestObject: diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index d10ce9d1fae..303f752ab0d 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -2025,3 +2025,26 @@ def test_event_decoder_reports_only_missing_aws_dependency(missing): assert error.value.__cause__ is failure else: assert error.value is failure + + +@pytest.mark.parametrize( + ("model", "base_model", "expected"), + [ + ( + "arn:aws:bedrock:us-east-1:000000000000:application-inference-profile/profile-id", + "bedrock/us.anthropic.claude-opus-4-8", + "us.anthropic.claude-opus-4-8", + ), + ( + "arn:aws:bedrock:us-east-1:000000000000:application-inference-profile/profile-id", + None, + "arn:aws:bedrock:us-east-1:000000000000:application-inference-profile/profile-id", + ), + ("us.anthropic.claude-opus-4-8", "anthropic.claude-sonnet-4-5", "us.anthropic.claude-opus-4-8"), + ], + ids=["profile-arn-with-base-model", "profile-arn-without-base-model", "named-model-keeps-its-name"], +) +def test_bedrock_capability_model_reads_base_model_only_for_application_profile_arns(model, base_model, expected): + from litellm.llms.bedrock.common_utils import bedrock_capability_model + + assert bedrock_capability_model(model, base_model) == expected