diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 03fbc35a97f..5281e6ae670 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -77,6 +77,7 @@ from litellm.llms.bedrock.common_utils import ( bedrock_model_supports_regex_lookaround, bedrock_native_output_format, bedrock_native_output_is_opt_in, + bedrock_native_output_retry_requested, bedrock_supports_tool_search, deployment_model_info, requests_output_config_format, @@ -1425,6 +1426,16 @@ class AmazonConverseConfig(BaseConfig): deployment_model_info(litellm_params), ) + def uses_native_output( + self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object] + ) -> bool: + return bedrock_native_output_retry_requested( + bedrock_capability_model(model, litellm_params.get("base_model")), + optional_params, + self.custom_llm_provider or "bedrock_converse", + deployment_model_info(litellm_params), + ) + def update_optional_params_with_thinking_tokens(self, non_default_params: dict, optional_params: dict): """ Handles scenario where max tokens is not specified. For anthropic models (anthropic api/bedrock/vertex ai), this requires having the max tokens being set and being greater than the thinking token budget. diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py index 62e1bee5217..1c38642e062 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py @@ -34,6 +34,7 @@ from litellm.llms.bedrock.common_utils import ( bedrock_capability_model, bedrock_chat_uses_native_structured_output, bedrock_cleaned_output_schema, + bedrock_native_output_retry_requested, bedrock_supports_tool_search, deployment_model_info, get_anthropic_beta_from_headers, @@ -161,6 +162,13 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): ) -> bool: return requests_opt_in_native_output(model, optional_params, deployment_model_info(litellm_params)) + def uses_native_output( + self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object] + ) -> bool: + return bedrock_native_output_retry_requested( + model, optional_params, model_info=deployment_model_info(litellm_params) + ) + def map_openai_params( self, non_default_params: dict, diff --git a/litellm/llms/bedrock/chat/native_output_fallback.py b/litellm/llms/bedrock/chat/native_output_fallback.py index 8069c3cf7e7..79742d0810c 100644 --- a/litellm/llms/bedrock/chat/native_output_fallback.py +++ b/litellm/llms/bedrock/chat/native_output_fallback.py @@ -1,4 +1,5 @@ from collections.abc import Callable, Coroutine +from dataclasses import dataclass from functools import partial from typing import Final, TypeAlias @@ -6,6 +7,7 @@ from litellm.llms.anthropic.common_utils import is_claude_strict_tools_rejection from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.common_utils import ( NativeOutputFallback, + bedrock_strict_tools_fallback, is_bedrock_native_output_rejection, warn_bedrock_native_output_rejected, warn_bedrock_strict_tools_dropped, @@ -17,7 +19,15 @@ NativeOutputResponse: TypeAlias = ModelResponse | CustomStreamWrapper NativeOutputAttemptResult: TypeAlias = NativeOutputResponse | Coroutine[object, object, NativeOutputResponse] NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeOutputAttemptResult] NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None] -_RetryFor: TypeAlias = Callable[[BaseLLMException], NativeOutputFallback | None] + + +@dataclass(frozen=True, slots=True) +class _SelectedFallback: + fallback: NativeOutputFallback + native_output_retry: bool + + +_RetryFor: TypeAlias = Callable[[BaseLLMException], _SelectedFallback | None] def _native_output_retry( @@ -47,17 +57,42 @@ def _fallback_for( fallback: NativeOutputFallbackFactory | None, strict_tools_fallback: NativeOutputFallbackFactory | None, model: str, -) -> NativeOutputFallback | None: +) -> _SelectedFallback | None: native_output_retry: Final = _native_output_retry(error, fallback, model) if native_output_retry is not None: - return native_output_retry - return _strict_tools_retry(error, strict_tools_fallback, model) + return _SelectedFallback(native_output_retry, native_output_retry=True) + strict_tools_retry: Final = _strict_tools_retry(error, strict_tools_fallback, model) + if strict_tools_retry is None: + return None + return _SelectedFallback(strict_tools_retry, native_output_retry=False) + + +def _strict_tools_retry_after_native_output_retry( + error: BaseLLMException, + retry: _SelectedFallback, + strict_tools_fallback: NativeOutputFallbackFactory | None, + model: str, +) -> NativeOutputFallback | None: + if ( + not retry.native_output_retry + or strict_tools_fallback is None + or not is_claude_strict_tools_rejection(error.status_code, error.message) + ): + return None + composed_retry: Final = bedrock_strict_tools_fallback( + model, retry.fallback.optional_params, retry.fallback.litellm_params + ) + if composed_retry is not None: + warn_bedrock_strict_tools_dropped(model) + return composed_retry async def _await_with_native_output_fallback( first_attempt: Coroutine[object, object, NativeOutputResponse], attempt: NativeOutputAttempt, retry_for: _RetryFor, + strict_tools_fallback: NativeOutputFallbackFactory | None, + model: str, ) -> NativeOutputResponse: try: return await first_attempt @@ -65,8 +100,17 @@ async def _await_with_native_output_fallback( retry: Final = retry_for(error) if retry is None: raise - second_attempt: Final = attempt(retry) - return await second_attempt if isinstance(second_attempt, Coroutine) else second_attempt + try: + second_attempt: Final = attempt(retry.fallback) + return await second_attempt if isinstance(second_attempt, Coroutine) else second_attempt + except BaseLLMException as error: + composed_retry: Final = _strict_tools_retry_after_native_output_retry( + error, retry, strict_tools_fallback, model + ) + if composed_retry is None: + raise + third_attempt: Final = attempt(composed_retry) + return await third_attempt if isinstance(third_attempt, Coroutine) else third_attempt def call_with_native_output_fallback( @@ -86,7 +130,15 @@ def call_with_native_output_fallback( retry: Final = retry_for(error) if retry is None: raise - return attempt(retry) + try: + return attempt(retry.fallback) + except BaseLLMException as retry_error: + composed_retry: Final = _strict_tools_retry_after_native_output_retry( + retry_error, retry, strict_tools_fallback, model + ) + if composed_retry is None: + raise + return attempt(composed_retry) if isinstance(first_attempt, Coroutine): - return _await_with_native_output_fallback(first_attempt, attempt, retry_for) + return _await_with_native_output_fallback(first_attempt, attempt, retry_for, strict_tools_fallback, model) return first_attempt diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index dfee28a7cc9..3385c89c3a7 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -292,6 +292,15 @@ def _bedrock_model_supports(model: str, key: str, custom_llm_provider: str = "be return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key=key) +def _bedrock_native_output_enabled( + model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None +) -> bool: + deployment_flag: Final = (model_info or {}).get("supports_native_structured_output") + if isinstance(deployment_flag, bool): + return deployment_flag + return _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) + + _DEPLOYMENT_MODEL_INFO: Final = TypeAdapter(dict[str, object]) @@ -312,9 +321,9 @@ def _bedrock_native_output_opt_in( def bedrock_supports_native_structured_output( model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None ) -> bool: - return _bedrock_model_supports( - model, "supports_native_structured_output", custom_llm_provider - ) or _bedrock_native_output_opt_in(model, custom_llm_provider, model_info) + return _bedrock_native_output_enabled(model, custom_llm_provider, model_info) or _bedrock_native_output_opt_in( + model, custom_llm_provider, model_info + ) def _bedrock_map_native_output_flag(model: str) -> object: @@ -349,7 +358,7 @@ def bedrock_chat_opts_into_native_structured_output( custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None, ) -> bool: - return _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) or ( + return _bedrock_native_output_enabled(model, custom_llm_provider, model_info) or ( thinking_requested and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info) ) @@ -370,15 +379,15 @@ def bedrock_chat_uses_native_structured_output( def bedrock_native_output_is_deployment_opt_in( model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None ) -> bool: - return not _bedrock_model_supports( - model, "supports_native_structured_output", custom_llm_provider - ) and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info) + return not _bedrock_native_output_enabled(model, custom_llm_provider, model_info) and _bedrock_native_output_opt_in( + model, custom_llm_provider, model_info + ) def bedrock_native_output_is_opt_in( model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None ) -> bool: - return not _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) and ( + return not _bedrock_native_output_enabled(model, custom_llm_provider, model_info) and ( _bedrock_native_output_opt_in(model, custom_llm_provider, model_info) or _bedrock_native_output_replaces_unforceable_tool(model, model_info) ) @@ -387,7 +396,21 @@ def bedrock_native_output_is_opt_in( def without_bedrock_native_output_opt_in(litellm_params: Mapping[str, object]) -> Mapping[str, object]: return { **litellm_params, - "model_info": {**deployment_model_info(litellm_params), "supports_bedrock_native_structured_output": False}, + "model_info": { + **deployment_model_info(litellm_params), + "supports_bedrock_native_structured_output": False, + }, + } + + +def for_bedrock_json_tool_retry(litellm_params: Mapping[str, object]) -> Mapping[str, object]: + opt_in_disabled: Final = without_bedrock_native_output_opt_in(litellm_params) + return { + **opt_in_disabled, + "model_info": { + **deployment_model_info(opt_in_disabled), + "supports_native_structured_output": False, + }, } @@ -395,6 +418,7 @@ _BEDROCK_NATIVE_OUTPUT_UNSUPPORTED: Final = "output_config.format: Extra inputs _BEDROCK_NATIVE_OUTPUT_REJECTIONS: Final = ( _BEDROCK_NATIVE_OUTPUT_UNSUPPORTED, "compilation exceeded limits: Schema is too complex", + "The compiled grammar is too large", "output_config.format.schema:", ) @@ -475,8 +499,27 @@ def requests_output_config_format(optional_params: Mapping[str, object]) -> bool def requests_native_output(anthropic_optional_params: Mapping[str, object]) -> bool: - return isinstance(anthropic_optional_params.get("output_format"), Mapping) or requests_output_config_format( - anthropic_optional_params + output_config: Final = json_object_or_none(anthropic_optional_params.get("outputConfig")) + return ( + isinstance(anthropic_optional_params.get("output_format"), Mapping) + or requests_output_config_format(anthropic_optional_params) + or ( + anthropic_optional_params.get("json_mode") is True + and output_config is not None + and isinstance(output_config.get("textFormat"), Mapping) + ) + ) + + +def bedrock_native_output_retry_requested( + model: str, + optional_params: Mapping[str, object], + custom_llm_provider: str = "bedrock", + model_info: Mapping[str, object] | None = None, +) -> bool: + return requests_native_output(optional_params) and ( + bedrock_native_output_is_opt_in(model, custom_llm_provider, model_info) + or _bedrock_native_output_enabled(model, custom_llm_provider, model_info) ) @@ -518,7 +561,7 @@ def bedrock_native_output_schema( model_info: Mapping[str, object] | None = None, ) -> Mapping[str, object] | None: native_schema: Final = bedrock_cleaned_output_schema(model, schema, custom_llm_provider) - if _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider): + if _bedrock_native_output_enabled(model, custom_llm_provider, model_info): return schema if native_schema is None else native_schema if _bedrock_native_output_opt_in(model, custom_llm_provider, model_info): return native_schema diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py index 8b4e2e5b5c5..51f37364245 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py @@ -40,15 +40,17 @@ from litellm.llms.bedrock.common_utils import ( BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS, BedrockError, apply_bedrock_invoke_structured_output, - bedrock_native_output_is_deployment_opt_in, + bedrock_native_output_format, + bedrock_native_output_retry_requested, bedrock_supports_tool_search, ensure_bedrock_anthropic_messages_tool_names, + for_bedrock_json_tool_retry, get_anthropic_beta_from_headers, is_bedrock_native_output_rejection, + json_object_or_none, normalize_bedrock_opus_output_config_effort, normalize_custom_field_on_tools, normalize_tool_input_schema_types_for_bedrock_invoke, - requests_native_output, requests_output_config_format, strip_unsupported_bedrock_invoke_output_config_keys, supports_bedrock_prompt_cache_ttl, @@ -133,13 +135,24 @@ class AmazonAnthropicClaudeMessagesConfig( ) -> GenericLiteLLMParams | None: model_info: Final = _MODEL_INFO.validate_python(litellm_params.model_info) if not ( - requests_native_output(anthropic_messages_optional_request_params) - and bedrock_native_output_is_deployment_opt_in(model, model_info=model_info) + bedrock_native_output_retry_requested( + model, anthropic_messages_optional_request_params, model_info=model_info + ) ): return None - return litellm_params.model_copy( - update={"model_info": {**(model_info or {}), "supports_bedrock_native_structured_output": False}} + legacy_output_format: Final = json_object_or_none( + anthropic_messages_optional_request_params.get("output_format") ) + output_config: Final = json_object_or_none(anthropic_messages_optional_request_params.get("output_config")) + output_config_format: Final = ( + json_object_or_none(output_config.get("format")) if output_config is not None else None + ) + output_format: Final = legacy_output_format if legacy_output_format is not None else output_config_format + if output_format is None or (bedrock_native_output_format(model, output_format, model_info=model_info) is None): + return None + retry_params: Final = for_bedrock_json_tool_retry({"model_info": model_info}) + retry_model_info: Final = _MODEL_INFO.validate_python(retry_params.get("model_info")) + return litellm_params.model_copy(update={"model_info": retry_model_info}) def should_retry_anthropic_messages_without_native_output( self, e: httpx.HTTPStatusError, request_body: Mapping[str, object], model: str diff --git a/litellm/main.py b/litellm/main.py index 8cb133bf76b..6b9964a450f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -143,7 +143,7 @@ from litellm.llms.bedrock.common_utils import ( bedrock_route_for_request, bedrock_strict_tools_fallback, deployment_model_info, - without_bedrock_native_output_opt_in, + for_bedrock_json_tool_retry, without_bedrock_route_prefix, ) from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config @@ -4369,7 +4369,7 @@ def _native_output_fallback( first_attempt_params: Mapping[str, object], first_attempt_litellm_params: Mapping[str, object], ) -> NativeOutputFallback: - litellm_params: Final = without_bedrock_native_output_opt_in(first_attempt_litellm_params) + litellm_params: Final = for_bedrock_json_tool_retry(first_attempt_litellm_params) remapped_params: Final = ctx.optional_params_for_model_info(deployment_model_info(litellm_params)) return NativeOutputFallback( optional_params={ @@ -4500,7 +4500,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu converse_attempt, ( partial(_converse_native_output_fallback, ctx, converse_params, dict(litellm_params)) - if AmazonConverseConfig().uses_opt_in_native_output(converse_model, converse_params, litellm_params) + if AmazonConverseConfig().uses_native_output(converse_model, converse_params, litellm_params) else None ), converse_model, @@ -4556,7 +4556,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu ( partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params)) if isinstance(provider_config, AmazonAnthropicClaudeConfig) - and provider_config.uses_opt_in_native_output(model, invoke_params, litellm_params) + and provider_config.uses_native_output(model, invoke_params, litellm_params) else None ), model, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a29f97ee9ea..ad57685da04 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2242,7 +2242,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2331,7 +2331,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2420,7 +2420,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2504,7 +2504,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2593,7 +2593,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2715,7 +2715,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2759,7 +2759,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2803,7 +2803,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2847,7 +2847,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2891,7 +2891,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -75504,7 +75504,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -75544,7 +75544,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a29f97ee9ea..ad57685da04 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -2242,7 +2242,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2331,7 +2331,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2420,7 +2420,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2504,7 +2504,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2593,7 +2593,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2715,7 +2715,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2759,7 +2759,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2803,7 +2803,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2847,7 +2847,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2891,7 +2891,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -75504,7 +75504,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -75544,7 +75544,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py index 93f6e237fc7..053574d7dd6 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py @@ -2047,8 +2047,9 @@ def _invoke_chat_params( ) @pytest.mark.parametrize("model", ["us.anthropic.claude-opus-4-8", "global.anthropic.claude-sonnet-5"]) def test_invoke_uses_opt_in_native_output_when_the_opt_in_drove_native_output( - local_model_cost_map, model, thinking_params + local_model_cost_map, model, thinking_params, monkeypatch ): + monkeypatch.setitem(litellm.model_cost[model], "supports_native_structured_output", False) optional_params: Final = _invoke_chat_params( model, None, {"response_format": _CITY_RESPONSE_FORMAT, **thinking_params} ) @@ -2083,6 +2084,15 @@ def test_invoke_uses_opt_in_native_output_is_false_when_native_output_did_not_co ) +def test_invoke_uses_default_native_output_for_schema_retry(local_model_cost_map: None) -> None: + model: Final = "us.anthropic.claude-opus-5" + optional_params: Final = _invoke_chat_params( + model, None, {"response_format": _CITY_RESPONSE_FORMAT, "thinking": {"type": "disabled"}} + ) + + assert AmazonAnthropicClaudeConfig().uses_native_output(model, optional_params, {}) + + @pytest.mark.parametrize( ("forced_tool_use", "map_flag", "model_info", "thinking_params", "native"), [ diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index b8eb5858fda..b37d6977c10 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -5244,6 +5244,44 @@ def test_models_whose_thinking_stops_under_native_output_keep_the_json_tool_only assert ("toolConfig" in request) is not native +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + ("model", "thinking_params", "native"), + [ + ("us.anthropic.claude-opus-5", None, True), + ("us.anthropic.claude-opus-5", {"thinking": {"type": "adaptive"}}, True), + ("us.anthropic.claude-opus-5", {"thinking": {"type": "disabled"}}, True), + ("us.anthropic.claude-opus-4-8", None, True), + ("us.anthropic.claude-opus-4-8", {"thinking": {"type": "adaptive"}}, True), + ("us.anthropic.claude-opus-4-8", {"thinking": {"type": "disabled"}}, True), + ("us-gov.anthropic.claude-opus-5", None, False), + ("us-gov.anthropic.claude-opus-4-8", None, False), + ], + ids=[ + "opus-5-unset", + "opus-5-adaptive", + "opus-5-disabled", + "opus-4-8-unset", + "opus-4-8-adaptive", + "opus-4-8-disabled", + "opus-5-us-gov", + "opus-4-8-us-gov", + ], +) +def test_opus_5_and_4_8_cost_map_defaults_use_native_output_across_thinking_modes( + model: str, thinking_params: Mapping[str, object] | None, native: bool +) -> None: + _, request = _converse_request_for_json_schema(model, False, thinking_params) + + if native: + assert request["outputConfig"]["textFormat"]["type"] == "json_schema" + assert "toolConfig" not in request + return + + assert "outputConfig" not in request + assert [tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]] == ["json_tool_call"] + + def _converse_capital_answer(request: httpx.Request) -> httpx.Response: return httpx.Response( 200, @@ -11599,7 +11637,10 @@ _OUTPUT_CONFIG_FORMAT: Final = { ids=["adaptive", "reasoning_effort", "budget", "output_config_format"], ) @pytest.mark.parametrize("model", ["us.anthropic.claude-opus-4-8", "global.anthropic.claude-sonnet-5"]) -def test_uses_opt_in_native_output_when_the_opt_in_drove_native_output(local_model_cost_map, model, request_params): +def test_uses_opt_in_native_output_when_the_opt_in_drove_native_output( + local_model_cost_map, monkeypatch, model, request_params +): + monkeypatch.setitem(litellm.model_cost[model], "supports_native_structured_output", False) optional_params: Final = _converse_params(model, None, request_params) assert AmazonConverseConfig().uses_opt_in_native_output(model, optional_params, {"model_info": {"id": "d1"}}) @@ -11630,7 +11671,10 @@ def test_uses_opt_in_native_output_is_false_when_native_output_did_not_come_from assert not AmazonConverseConfig().uses_opt_in_native_output(model, optional_params, {"model_info": model_info}) -def test_uses_opt_in_native_output_reads_the_base_model_of_an_application_inference_profile(local_model_cost_map): +def test_uses_opt_in_native_output_reads_the_base_model_of_an_application_inference_profile( + local_model_cost_map, monkeypatch +): + monkeypatch.setitem(litellm.model_cost["us.anthropic.claude-opus-4-8"], "supports_native_structured_output", False) profile: Final = "arn:aws:bedrock:us-east-1:000000000000:application-inference-profile/profile-id" litellm_params: Final = {"base_model": "bedrock/us.anthropic.claude-opus-4-8"} optional_params: Final = _converse_params( @@ -11698,8 +11742,9 @@ _BEDROCK_SCHEMA_REFUSAL: Final = "output_config.format.schema: pattern is too co ) @pytest.mark.respx(assert_all_called=True) def test_a_refused_native_schema_is_sent_again_only_when_the_retry_drops_native_output( - caller_params: dict, expected_calls: int, respx_mock: respx.Router, local_model_cost_map: None + caller_params: dict, expected_calls: int, respx_mock: respx.Router, local_model_cost_map: None, monkeypatch ) -> None: + monkeypatch.setitem(litellm.model_cost["us.anthropic.claude-opus-4-8"], "supports_native_structured_output", False) route: Final = respx_mock.post(url__regex=r"/converse$").mock( return_value=httpx.Response(400, json={"message": _BEDROCK_SCHEMA_REFUSAL}) ) @@ -11722,6 +11767,61 @@ def test_a_refused_native_schema_is_sent_again_only_when_the_retry_drops_native_ assert all("outputConfig" not in json.loads(call.request.content) for call in route.calls[1:]) +@pytest.mark.respx(assert_all_called=True) +def test_a_default_native_schema_rejection_retries_with_the_json_tool( + respx_mock: respx.Router, local_model_cost_map: None +) -> None: + route: Final = respx_mock.post(url__regex=r"/converse$").mock( + return_value=httpx.Response(400, json={"message": _BEDROCK_SCHEMA_REFUSAL}) + ) + + with pytest.raises(litellm.BadRequestError, match=re.escape(_BEDROCK_SCHEMA_REFUSAL)): + completion( + model="bedrock/converse/us.anthropic.claude-opus-5", + messages=[{"role": "user", "content": "What is the capital of France?"}], + response_format=CAPITAL_RESPONSE_FORMAT, + thinking={"type": "disabled"}, + num_retries=0, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + client=HTTPHandler(), + ) + + assert route.call_count == 2 + first_request: Final = json.loads(route.calls[0].request.content) + retry_request: Final = json.loads(route.calls[1].request.content) + assert "outputConfig" in first_request and "toolConfig" not in first_request + assert "toolConfig" in retry_request and "outputConfig" not in retry_request + + +@pytest.mark.respx(assert_all_called=True) +def test_an_unforceable_tool_native_schema_rejection_retries_with_the_json_tool( + respx_mock: respx.Router, local_model_cost_map: None +) -> None: + route: Final = respx_mock.post(url__regex=r"/converse$").mock( + return_value=httpx.Response(400, json={"message": _BEDROCK_SCHEMA_REFUSAL}) + ) + + with pytest.raises(litellm.BadRequestError, match=re.escape(_BEDROCK_SCHEMA_REFUSAL)): + completion( + model="bedrock/converse/anthropic.claude-fable-5-1", + messages=[{"role": "user", "content": "What is the capital of France?"}], + response_format=CAPITAL_RESPONSE_FORMAT, + num_retries=0, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + client=HTTPHandler(), + ) + + assert route.call_count == 2 + first_request: Final = json.loads(route.calls[0].request.content) + retry_request: Final = json.loads(route.calls[1].request.content) + assert "outputConfig" in first_request and "toolConfig" not in first_request + assert "toolConfig" in retry_request and "outputConfig" not in retry_request + + CONTROL_PLANE_ENDPOINT: Final = "https://bedrock-fips.us-east-1.amazonaws.com" diff --git a/tests/unit/llms/bedrock/chat/test_invoke_handler.py b/tests/unit/llms/bedrock/chat/test_invoke_handler.py index 4424f7640a3..2faa089e1c3 100644 --- a/tests/unit/llms/bedrock/chat/test_invoke_handler.py +++ b/tests/unit/llms/bedrock/chat/test_invoke_handler.py @@ -1214,7 +1214,7 @@ async def test_converse_json_schema_stream_sends_claude_prose_only_when_claude_s transport: Final = httpx.MockTransport(handle_request) request: Final = { - "model": "bedrock/us.anthropic.claude-opus-5", + "model": "bedrock/anthropic.claude-opus-5", "messages": [{"role": "user", "content": "What is the LCM of 24 and 120? Reply in JSON."}], "response_format": { "type": "json_schema", diff --git a/tests/unit/llms/bedrock/chat/test_native_output_fallback.py b/tests/unit/llms/bedrock/chat/test_native_output_fallback.py index 9ee8c14f5d6..81832d7d8bf 100644 --- a/tests/unit/llms/bedrock/chat/test_native_output_fallback.py +++ b/tests/unit/llms/bedrock/chat/test_native_output_fallback.py @@ -13,10 +13,33 @@ _MODEL: Final = "us.anthropic.claude-opus-4-8" _FALLBACK: Final = NativeOutputFallback( optional_params={"tools": "json_tool_call"}, litellm_params={"model_info": {"opted_out": True}} ) +_NATIVE_RETRY_WITH_STRICT_TOOLS: Final = NativeOutputFallback( + optional_params={ + "tools": ( + {"function": {"name": "json_schema", "strict": True}}, + {"function": {"name": "custom", "strict": True}}, + ) + }, + litellm_params={ + "model_info": { + "supports_native_structured_output": False, + "supports_bedrock_native_structured_output": False, + } + }, +) +_COMPOSED_FALLBACK: Final = NativeOutputFallback( + optional_params={"tools": [{"function": {"name": "json_schema"}}, {"function": {"name": "custom"}}]}, + litellm_params=_NATIVE_RETRY_WITH_STRICT_TOOLS.litellm_params, +) _REJECTION: Final = BedrockError( status_code=400, message='{"message":"The model returned the following errors: output_config.format: Extra inputs are not permitted"}', ) +_GRAMMAR_REJECTION: Final = BedrockError( + status_code=400, + message='{"message":"The model returned the following errors: The compiled grammar is too large, which would cause ' + 'performance issues. Simplify your tool schemas or reduce the number of strict tools."}', +) _SCHEMA_REFUSAL: Final = BedrockError( status_code=400, message='{"message":"output_config.format.schema: pattern is too complex for structured output"}' ) @@ -58,8 +81,15 @@ _MODES: Final = pytest.mark.parametrize("asynchronous", [False, True], ids=["syn @_MODES -def test_a_native_output_rejection_is_sent_again_once_with_the_fallback_params(asynchronous: bool) -> None: - bedrock: Final = ScriptedBedrock(errors=(_REJECTION, None), asynchronous=asynchronous) +@pytest.mark.parametrize( + "rejection", + [_REJECTION, _GRAMMAR_REJECTION], + ids=["format_not_permitted", "compiled_grammar_too_large"], +) +def test_a_native_output_rejection_is_sent_again_once_with_the_fallback_params( + asynchronous: bool, rejection: BedrockError +) -> None: + bedrock: Final = ScriptedBedrock(errors=(rejection, None), asynchronous=asynchronous) response: Final = _call(bedrock, _FALLBACK) @@ -77,6 +107,19 @@ def test_a_second_native_output_rejection_reaches_the_caller(asynchronous: bool) assert len(bedrock.sent) == 2 +@_MODES +def test_a_grammar_rejection_after_a_native_retry_reaches_the_caller_without_strict_tools_fallback( + asynchronous: bool, +) -> None: + bedrock: Final = ScriptedBedrock(errors=(_GRAMMAR_REJECTION, _GRAMMAR_REJECTION), asynchronous=asynchronous) + + with pytest.raises(BedrockError) as raised: + _resolve(call_with_native_output_fallback(bedrock, lambda: _FALLBACK, _MODEL)) + + assert raised.value is _GRAMMAR_REJECTION + assert bedrock.sent == (None, _FALLBACK) + + @_MODES @pytest.mark.parametrize( ("error", "fallback"), @@ -155,13 +198,16 @@ _OTHER_400S_WITH_A_STRICT_TOOL: Final = tuple( def _call_with_strict_tools_fallback( - bedrock: ScriptedBedrock, fallback: NativeOutputFallback | None, strict_tools_fallback: NativeOutputFallback | None + bedrock: ScriptedBedrock, + fallback: NativeOutputFallback | None, + strict_tools_fallback: NativeOutputFallback | None, + model: str = _MODEL, ) -> object: return _resolve( call_with_native_output_fallback( bedrock, None if fallback is None else lambda: fallback, - _MODEL, + model, strict_tools_fallback=None if strict_tools_fallback is None else lambda: strict_tools_fallback, ) ) @@ -223,3 +269,18 @@ def test_each_rejection_is_sent_again_with_its_own_fallback_when_both_are_set( _call_with_strict_tools_fallback(bedrock, _FALLBACK, _STRICT_FALLBACK) assert bedrock.sent == (None, retried_with) + + +@_MODES +def test_a_strict_tools_rejection_after_native_retry_composes_both_fallbacks(asynchronous: bool) -> None: + bedrock: Final = ScriptedBedrock( + errors=(_GRAMMAR_REJECTION, _GRAMMAR_REJECTION, None), + asynchronous=asynchronous, + ) + + response: Final = _call_with_strict_tools_fallback( + bedrock, _NATIVE_RETRY_WITH_STRICT_TOOLS, _STRICT_FALLBACK, "us.anthropic.claude-sonnet-4-6" + ) + + assert isinstance(response, ModelResponse) and response.id == "answer-3" + assert bedrock.sent == (None, _NATIVE_RETRY_WITH_STRICT_TOOLS, _COMPOSED_FALLBACK) diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index 512e1f325f4..a9edbd69ee3 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -2047,7 +2047,7 @@ class TestBedrockBatchAnthropicRowParams: @pytest.mark.parametrize( ("model", "native_output"), - [("bedrock/us.anthropic.claude-opus-4-8", False), (MODEL, True)], + [("bedrock/us.anthropic.claude-sonnet-5-5", False), (MODEL, True)], ids=["opt_in_native_output_model", "native_output_model"], ) def test_thinking_json_schema_rows_use_native_output_only_on_models_that_always_take_it( diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py index 07fabbf2393..ff4e3650464 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py @@ -4016,7 +4016,10 @@ _NATIVE_OUTPUT_REJECTION: Final = { "message": "The model returned the following errors: compilation exceeded limits: Schema is too complex." } _STALE_SIGNATURE_ERROR: Final = {"message": "messages.1.content.0: Invalid `signature` in `thinking` block"} -_OPTED_OUT: Final = {"supports_bedrock_native_structured_output": False} +_OPTED_OUT: Final = { + "supports_native_structured_output": False, + "supports_bedrock_native_structured_output": False, +} _CONSTRAINED_SCHEMA: Final = { "type": "object", "properties": { @@ -4158,21 +4161,21 @@ async def test_bedrock_messages_native_output_rejection_and_signature_400_each_g @pytest.mark.usefixtures("local_model_cost_map") @pytest.mark.parametrize( - ("model", "model_info", "error"), + ("model", "model_info", "error", "should_retry"), [ - ("us.anthropic.claude-opus-4-8", None, {"message": "messages.0.content: Field required"}), - ("us.anthropic.claude-sonnet-4-6", None, _NATIVE_OUTPUT_REJECTION), - ("us.anthropic.claude-opus-4-8", _OPTED_OUT, _NATIVE_OUTPUT_REJECTION), + ("us.anthropic.claude-opus-4-8", None, {"message": "messages.0.content: Field required"}, False), + ("us.anthropic.claude-sonnet-4-6", None, _NATIVE_OUTPUT_REJECTION, True), + ("us.anthropic.claude-opus-4-8", _OPTED_OUT, _NATIVE_OUTPUT_REJECTION, False), ], - ids=["other_400", "native_model", "deployment_opt_out"], + ids=["other_400", "native_rejection", "deployment_opt_out"], ) -async def test_bedrock_messages_400_is_not_retried_unless_it_rejects_opt_in_native_output( - model: str, model_info: Mapping[str, object] | None, error: Mapping[str, object] +async def test_bedrock_messages_retries_only_native_output_rejections( + model: str, model_info: Mapping[str, object] | None, error: Mapping[str, object], should_retry: bool ) -> None: bedrock, raised = await _send_bedrock_messages(model, model_info, [(400, error), (200, _ANTHROPIC_MESSAGE)]) - assert isinstance(raised, litellm.BadRequestError) - assert len(bedrock.bodies) == 1 + assert (raised is None) is should_retry + assert len(bedrock.bodies) == (2 if should_retry else 1) @pytest.mark.usefixtures("local_model_cost_map") diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index 80715105b7f..d8e9726358c 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -13,6 +13,7 @@ from litellm.llms.bedrock.common_utils import( bedrock_strict_tools_fallback, deployment_model_info, extract_model_name_from_bedrock_arn, + for_bedrock_json_tool_retry, get_bedrock_base_model, get_bedrock_cross_region_inference_regions, is_bedrock_native_output_rejection, @@ -2328,6 +2329,10 @@ def test_bedrock_capability_model_reads_base_model_only_for_application_profile_ _FORMAT_REJECTION: Final = "The model returned the following errors: output_config.format: Extra inputs are not permitted" _COMPILE_REJECTION: Final = "The model returned the following errors: compilation exceeded limits: Schema is too complex." +_GRAMMAR_REJECTION: Final = ( + "The model returned the following errors: The compiled grammar is too large, which would cause performance issues. " + "Simplify your tool schemas or reduce the number of strict tools." +) _SCHEMA_PATTERN_REJECTION: Final = ( "The model returned the following errors: output_config.format.schema: Unsupported regex feature in pattern " "field: pattern is too complex for structured output: reduce the {n,m} upper bound, narrow the character class " @@ -2340,6 +2345,7 @@ _SCHEMA_PATTERN_REJECTION: Final = ( [ (400, f'{{"message":"{_FORMAT_REJECTION}"}}', True), (400, f'{{"message":"{_COMPILE_REJECTION}"}}', True), + (400, _GRAMMAR_REJECTION, True), (400, f'{{"message":"{_SCHEMA_PATTERN_REJECTION}"}}', True), (400, "Malformed input request: #: extraneous key [output_config] is not permitted", False), (400, "The model returned the following errors: messages.0.content: Field required", False), @@ -2349,6 +2355,7 @@ _SCHEMA_PATTERN_REJECTION: Final = ( ids=[ "format_not_permitted", "schema_too_complex", + "compiled_grammar_too_large", "schema_pattern_too_complex", "other_key_not_permitted", "other_400", @@ -2365,7 +2372,7 @@ def test_is_bedrock_native_output_rejection_matches_only_the_400s_that_reject_na @pytest.mark.parametrize( ("model", "model_info", "expected"), [ - ("us.anthropic.claude-opus-4-8", None, True), + ("us.anthropic.claude-opus-4-8", None, False), ("global.anthropic.claude-sonnet-5", None, True), ("us-gov.anthropic.claude-opus-4-8", None, False), ("us.anthropic.claude-opus-4-8", {"supports_bedrock_native_structured_output": False}, False), @@ -2389,9 +2396,43 @@ def test_without_bedrock_native_output_opt_in_turns_the_opt_in_off_and_keeps_the "us.anthropic.claude-opus-4-8", model_info=deployment_model_info(turned_off) ) assert turned_off["api_base"] == "https://bedrock.example" - assert deployment_model_info(turned_off)["id"] == "deployment-1" + assert deployment_model_info(turned_off) == { + "id": "deployment-1", + "supports_bedrock_native_structured_output": False, + } assert litellm_params["model_info"] == {"id": "deployment-1"} + native_default: Final = without_bedrock_native_output_opt_in( + {"model_info": {"supports_native_structured_output": True}} + ) + assert deployment_model_info(native_default) == { + "supports_native_structured_output": True, + "supports_bedrock_native_structured_output": False, + } + + +def test_for_bedrock_json_tool_retry_disables_both_native_output_flags(local_model_cost_map: None) -> None: + litellm_params: Final = { + "model_info": { + "id": "deployment-1", + "supports_native_structured_output": True, + "supports_bedrock_native_structured_output": True, + } + } + + retry_params: Final = for_bedrock_json_tool_retry(litellm_params) + + assert deployment_model_info(retry_params) == { + "id": "deployment-1", + "supports_native_structured_output": False, + "supports_bedrock_native_structured_output": False, + } + assert litellm_params["model_info"] == { + "id": "deployment-1", + "supports_native_structured_output": True, + "supports_bedrock_native_structured_output": True, + } + def test_bedrock_response_stream_shape_is_unavailable_when_botocore_describes_the_service_as_a_list(): import litellm.llms.bedrock.common_utils as mod diff --git a/tests/unit/test_utils_get_model_info.py b/tests/unit/test_utils_get_model_info.py index 302a7d60cdd..d08742be4af 100644 --- a/tests/unit/test_utils_get_model_info.py +++ b/tests/unit/test_utils_get_model_info.py @@ -332,6 +332,18 @@ def _cross_region_profiles( ) +_BEDROCK_PROFILE_ONLY_NATIVE_OUTPUT_MODELS: Final = frozenset( + { + "anthropic.claude-opus-5-5", + "anthropic.claude-fable-5-1", + "anthropic.claude-sonnet-5", + "anthropic.claude-haiku-5-5", + "anthropic.claude-opus-5", + "anthropic.claude-opus-4-8", + } +) + + def test_get_model_info_bedrock_cross_region_capability_parity(): """ Cross-region inference profiles carry litellm_provider "bedrock_converse", so the @@ -362,6 +374,13 @@ def test_get_model_info_bedrock_cross_region_capability_parity(): if _govcloud_may_lag(k, cap): assert base_value or not v[cap], f"{cap} is on {k} but not on {base_model_key}" continue + if ( + cap == "supports_native_structured_output" + and base_model_key in _BEDROCK_PROFILE_ONLY_NATIVE_OUTPUT_MODELS + and v[cap] is True + ): + assert base_value is False, f"{cap} is already enabled on bare model {base_model_key}" + continue assert ( v[cap] == base_value ), f"{cap} is {v[cap]} on {k} but {base_value} on {base_model_key}"