mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-07 07:58:09 +00:00
feat(bedrock): default to native structured output on Opus 5 and Opus 4.8 (#50222)
* feat(bedrock): enable native structured output for Opus 5 models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): keep govcloud and bare model rows off native output Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): retry rejected native schemas with the JSON tool on default-native models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): keep the rejection retry on the unforceable-tool native path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): retry rejected schemas on the Messages path for default-native models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(bedrock): keep JSON-tool streaming coverage on unchanged route Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(bedrock): format Messages native output retry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(bedrock): allow profile-only native output metadata Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): only disable default-native output on schema-rejection retries Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(bedrock): type fixture parameters in native output retry tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): retry with the JSON tool when Bedrock rejects a schema's grammar size Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): drop strict tools after a native-output retry hits the grammar limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(bedrock): format native output fallback Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Mateo Wang
GitHub
parent
7b550cdd94
commit
14d50ed40d
16 changed files with 437 additions and 76 deletions
|
|
@ -77,6 +77,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
bedrock_model_supports_regex_lookaround,
|
||||
bedrock_native_output_format,
|
||||
bedrock_native_output_is_opt_in,
|
||||
bedrock_native_output_retry_requested,
|
||||
bedrock_supports_tool_search,
|
||||
deployment_model_info,
|
||||
requests_output_config_format,
|
||||
|
|
@ -1425,6 +1426,16 @@ class AmazonConverseConfig(BaseConfig):
|
|||
deployment_model_info(litellm_params),
|
||||
)
|
||||
|
||||
def uses_native_output(
|
||||
self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
|
||||
) -> bool:
|
||||
return bedrock_native_output_retry_requested(
|
||||
bedrock_capability_model(model, litellm_params.get("base_model")),
|
||||
optional_params,
|
||||
self.custom_llm_provider or "bedrock_converse",
|
||||
deployment_model_info(litellm_params),
|
||||
)
|
||||
|
||||
def update_optional_params_with_thinking_tokens(self, non_default_params: dict, optional_params: dict):
|
||||
"""
|
||||
Handles scenario where max tokens is not specified. For anthropic models (anthropic api/bedrock/vertex ai), this requires having the max tokens being set and being greater than the thinking token budget.
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
bedrock_capability_model,
|
||||
bedrock_chat_uses_native_structured_output,
|
||||
bedrock_cleaned_output_schema,
|
||||
bedrock_native_output_retry_requested,
|
||||
bedrock_supports_tool_search,
|
||||
deployment_model_info,
|
||||
get_anthropic_beta_from_headers,
|
||||
|
|
@ -161,6 +162,13 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
) -> bool:
|
||||
return requests_opt_in_native_output(model, optional_params, deployment_model_info(litellm_params))
|
||||
|
||||
def uses_native_output(
|
||||
self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
|
||||
) -> bool:
|
||||
return bedrock_native_output_retry_requested(
|
||||
model, optional_params, model_info=deployment_model_info(litellm_params)
|
||||
)
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from collections.abc import Callable, Coroutine
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
|
|
@ -6,6 +7,7 @@ from litellm.llms.anthropic.common_utils import is_claude_strict_tools_rejection
|
|||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
NativeOutputFallback,
|
||||
bedrock_strict_tools_fallback,
|
||||
is_bedrock_native_output_rejection,
|
||||
warn_bedrock_native_output_rejected,
|
||||
warn_bedrock_strict_tools_dropped,
|
||||
|
|
@ -17,7 +19,15 @@ NativeOutputResponse: TypeAlias = ModelResponse | CustomStreamWrapper
|
|||
NativeOutputAttemptResult: TypeAlias = NativeOutputResponse | Coroutine[object, object, NativeOutputResponse]
|
||||
NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeOutputAttemptResult]
|
||||
NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None]
|
||||
_RetryFor: TypeAlias = Callable[[BaseLLMException], NativeOutputFallback | None]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SelectedFallback:
|
||||
fallback: NativeOutputFallback
|
||||
native_output_retry: bool
|
||||
|
||||
|
||||
_RetryFor: TypeAlias = Callable[[BaseLLMException], _SelectedFallback | None]
|
||||
|
||||
|
||||
def _native_output_retry(
|
||||
|
|
@ -47,17 +57,42 @@ def _fallback_for(
|
|||
fallback: NativeOutputFallbackFactory | None,
|
||||
strict_tools_fallback: NativeOutputFallbackFactory | None,
|
||||
model: str,
|
||||
) -> NativeOutputFallback | None:
|
||||
) -> _SelectedFallback | None:
|
||||
native_output_retry: Final = _native_output_retry(error, fallback, model)
|
||||
if native_output_retry is not None:
|
||||
return native_output_retry
|
||||
return _strict_tools_retry(error, strict_tools_fallback, model)
|
||||
return _SelectedFallback(native_output_retry, native_output_retry=True)
|
||||
strict_tools_retry: Final = _strict_tools_retry(error, strict_tools_fallback, model)
|
||||
if strict_tools_retry is None:
|
||||
return None
|
||||
return _SelectedFallback(strict_tools_retry, native_output_retry=False)
|
||||
|
||||
|
||||
def _strict_tools_retry_after_native_output_retry(
|
||||
error: BaseLLMException,
|
||||
retry: _SelectedFallback,
|
||||
strict_tools_fallback: NativeOutputFallbackFactory | None,
|
||||
model: str,
|
||||
) -> NativeOutputFallback | None:
|
||||
if (
|
||||
not retry.native_output_retry
|
||||
or strict_tools_fallback is None
|
||||
or not is_claude_strict_tools_rejection(error.status_code, error.message)
|
||||
):
|
||||
return None
|
||||
composed_retry: Final = bedrock_strict_tools_fallback(
|
||||
model, retry.fallback.optional_params, retry.fallback.litellm_params
|
||||
)
|
||||
if composed_retry is not None:
|
||||
warn_bedrock_strict_tools_dropped(model)
|
||||
return composed_retry
|
||||
|
||||
|
||||
async def _await_with_native_output_fallback(
|
||||
first_attempt: Coroutine[object, object, NativeOutputResponse],
|
||||
attempt: NativeOutputAttempt,
|
||||
retry_for: _RetryFor,
|
||||
strict_tools_fallback: NativeOutputFallbackFactory | None,
|
||||
model: str,
|
||||
) -> NativeOutputResponse:
|
||||
try:
|
||||
return await first_attempt
|
||||
|
|
@ -65,8 +100,17 @@ async def _await_with_native_output_fallback(
|
|||
retry: Final = retry_for(error)
|
||||
if retry is None:
|
||||
raise
|
||||
second_attempt: Final = attempt(retry)
|
||||
return await second_attempt if isinstance(second_attempt, Coroutine) else second_attempt
|
||||
try:
|
||||
second_attempt: Final = attempt(retry.fallback)
|
||||
return await second_attempt if isinstance(second_attempt, Coroutine) else second_attempt
|
||||
except BaseLLMException as error:
|
||||
composed_retry: Final = _strict_tools_retry_after_native_output_retry(
|
||||
error, retry, strict_tools_fallback, model
|
||||
)
|
||||
if composed_retry is None:
|
||||
raise
|
||||
third_attempt: Final = attempt(composed_retry)
|
||||
return await third_attempt if isinstance(third_attempt, Coroutine) else third_attempt
|
||||
|
||||
|
||||
def call_with_native_output_fallback(
|
||||
|
|
@ -86,7 +130,15 @@ def call_with_native_output_fallback(
|
|||
retry: Final = retry_for(error)
|
||||
if retry is None:
|
||||
raise
|
||||
return attempt(retry)
|
||||
try:
|
||||
return attempt(retry.fallback)
|
||||
except BaseLLMException as retry_error:
|
||||
composed_retry: Final = _strict_tools_retry_after_native_output_retry(
|
||||
retry_error, retry, strict_tools_fallback, model
|
||||
)
|
||||
if composed_retry is None:
|
||||
raise
|
||||
return attempt(composed_retry)
|
||||
if isinstance(first_attempt, Coroutine):
|
||||
return _await_with_native_output_fallback(first_attempt, attempt, retry_for)
|
||||
return _await_with_native_output_fallback(first_attempt, attempt, retry_for, strict_tools_fallback, model)
|
||||
return first_attempt
|
||||
|
|
|
|||
|
|
@ -292,6 +292,15 @@ def _bedrock_model_supports(model: str, key: str, custom_llm_provider: str = "be
|
|||
return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key=key)
|
||||
|
||||
|
||||
def _bedrock_native_output_enabled(
|
||||
model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None
|
||||
) -> bool:
|
||||
deployment_flag: Final = (model_info or {}).get("supports_native_structured_output")
|
||||
if isinstance(deployment_flag, bool):
|
||||
return deployment_flag
|
||||
return _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider)
|
||||
|
||||
|
||||
_DEPLOYMENT_MODEL_INFO: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
|
|
@ -312,9 +321,9 @@ def _bedrock_native_output_opt_in(
|
|||
def bedrock_supports_native_structured_output(
|
||||
model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
|
||||
) -> bool:
|
||||
return _bedrock_model_supports(
|
||||
model, "supports_native_structured_output", custom_llm_provider
|
||||
) or _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
|
||||
return _bedrock_native_output_enabled(model, custom_llm_provider, model_info) or _bedrock_native_output_opt_in(
|
||||
model, custom_llm_provider, model_info
|
||||
)
|
||||
|
||||
|
||||
def _bedrock_map_native_output_flag(model: str) -> object:
|
||||
|
|
@ -349,7 +358,7 @@ def bedrock_chat_opts_into_native_structured_output(
|
|||
custom_llm_provider: str = "bedrock",
|
||||
model_info: Mapping[str, object] | None = None,
|
||||
) -> bool:
|
||||
return _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) or (
|
||||
return _bedrock_native_output_enabled(model, custom_llm_provider, model_info) or (
|
||||
thinking_requested and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
|
||||
)
|
||||
|
||||
|
|
@ -370,15 +379,15 @@ def bedrock_chat_uses_native_structured_output(
|
|||
def bedrock_native_output_is_deployment_opt_in(
|
||||
model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
|
||||
) -> bool:
|
||||
return not _bedrock_model_supports(
|
||||
model, "supports_native_structured_output", custom_llm_provider
|
||||
) and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
|
||||
return not _bedrock_native_output_enabled(model, custom_llm_provider, model_info) and _bedrock_native_output_opt_in(
|
||||
model, custom_llm_provider, model_info
|
||||
)
|
||||
|
||||
|
||||
def bedrock_native_output_is_opt_in(
|
||||
model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
|
||||
) -> bool:
|
||||
return not _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) and (
|
||||
return not _bedrock_native_output_enabled(model, custom_llm_provider, model_info) and (
|
||||
_bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
|
||||
or _bedrock_native_output_replaces_unforceable_tool(model, model_info)
|
||||
)
|
||||
|
|
@ -387,7 +396,21 @@ def bedrock_native_output_is_opt_in(
|
|||
def without_bedrock_native_output_opt_in(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {
|
||||
**litellm_params,
|
||||
"model_info": {**deployment_model_info(litellm_params), "supports_bedrock_native_structured_output": False},
|
||||
"model_info": {
|
||||
**deployment_model_info(litellm_params),
|
||||
"supports_bedrock_native_structured_output": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def for_bedrock_json_tool_retry(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
opt_in_disabled: Final = without_bedrock_native_output_opt_in(litellm_params)
|
||||
return {
|
||||
**opt_in_disabled,
|
||||
"model_info": {
|
||||
**deployment_model_info(opt_in_disabled),
|
||||
"supports_native_structured_output": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -395,6 +418,7 @@ _BEDROCK_NATIVE_OUTPUT_UNSUPPORTED: Final = "output_config.format: Extra inputs
|
|||
_BEDROCK_NATIVE_OUTPUT_REJECTIONS: Final = (
|
||||
_BEDROCK_NATIVE_OUTPUT_UNSUPPORTED,
|
||||
"compilation exceeded limits: Schema is too complex",
|
||||
"The compiled grammar is too large",
|
||||
"output_config.format.schema:",
|
||||
)
|
||||
|
||||
|
|
@ -475,8 +499,27 @@ def requests_output_config_format(optional_params: Mapping[str, object]) -> bool
|
|||
|
||||
|
||||
def requests_native_output(anthropic_optional_params: Mapping[str, object]) -> bool:
|
||||
return isinstance(anthropic_optional_params.get("output_format"), Mapping) or requests_output_config_format(
|
||||
anthropic_optional_params
|
||||
output_config: Final = json_object_or_none(anthropic_optional_params.get("outputConfig"))
|
||||
return (
|
||||
isinstance(anthropic_optional_params.get("output_format"), Mapping)
|
||||
or requests_output_config_format(anthropic_optional_params)
|
||||
or (
|
||||
anthropic_optional_params.get("json_mode") is True
|
||||
and output_config is not None
|
||||
and isinstance(output_config.get("textFormat"), Mapping)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def bedrock_native_output_retry_requested(
|
||||
model: str,
|
||||
optional_params: Mapping[str, object],
|
||||
custom_llm_provider: str = "bedrock",
|
||||
model_info: Mapping[str, object] | None = None,
|
||||
) -> bool:
|
||||
return requests_native_output(optional_params) and (
|
||||
bedrock_native_output_is_opt_in(model, custom_llm_provider, model_info)
|
||||
or _bedrock_native_output_enabled(model, custom_llm_provider, model_info)
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -518,7 +561,7 @@ def bedrock_native_output_schema(
|
|||
model_info: Mapping[str, object] | None = None,
|
||||
) -> Mapping[str, object] | None:
|
||||
native_schema: Final = bedrock_cleaned_output_schema(model, schema, custom_llm_provider)
|
||||
if _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider):
|
||||
if _bedrock_native_output_enabled(model, custom_llm_provider, model_info):
|
||||
return schema if native_schema is None else native_schema
|
||||
if _bedrock_native_output_opt_in(model, custom_llm_provider, model_info):
|
||||
return native_schema
|
||||
|
|
|
|||
|
|
@ -40,15 +40,17 @@ from litellm.llms.bedrock.common_utils import (
|
|||
BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS,
|
||||
BedrockError,
|
||||
apply_bedrock_invoke_structured_output,
|
||||
bedrock_native_output_is_deployment_opt_in,
|
||||
bedrock_native_output_format,
|
||||
bedrock_native_output_retry_requested,
|
||||
bedrock_supports_tool_search,
|
||||
ensure_bedrock_anthropic_messages_tool_names,
|
||||
for_bedrock_json_tool_retry,
|
||||
get_anthropic_beta_from_headers,
|
||||
is_bedrock_native_output_rejection,
|
||||
json_object_or_none,
|
||||
normalize_bedrock_opus_output_config_effort,
|
||||
normalize_custom_field_on_tools,
|
||||
normalize_tool_input_schema_types_for_bedrock_invoke,
|
||||
requests_native_output,
|
||||
requests_output_config_format,
|
||||
strip_unsupported_bedrock_invoke_output_config_keys,
|
||||
supports_bedrock_prompt_cache_ttl,
|
||||
|
|
@ -133,13 +135,24 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
) -> GenericLiteLLMParams | None:
|
||||
model_info: Final = _MODEL_INFO.validate_python(litellm_params.model_info)
|
||||
if not (
|
||||
requests_native_output(anthropic_messages_optional_request_params)
|
||||
and bedrock_native_output_is_deployment_opt_in(model, model_info=model_info)
|
||||
bedrock_native_output_retry_requested(
|
||||
model, anthropic_messages_optional_request_params, model_info=model_info
|
||||
)
|
||||
):
|
||||
return None
|
||||
return litellm_params.model_copy(
|
||||
update={"model_info": {**(model_info or {}), "supports_bedrock_native_structured_output": False}}
|
||||
legacy_output_format: Final = json_object_or_none(
|
||||
anthropic_messages_optional_request_params.get("output_format")
|
||||
)
|
||||
output_config: Final = json_object_or_none(anthropic_messages_optional_request_params.get("output_config"))
|
||||
output_config_format: Final = (
|
||||
json_object_or_none(output_config.get("format")) if output_config is not None else None
|
||||
)
|
||||
output_format: Final = legacy_output_format if legacy_output_format is not None else output_config_format
|
||||
if output_format is None or (bedrock_native_output_format(model, output_format, model_info=model_info) is None):
|
||||
return None
|
||||
retry_params: Final = for_bedrock_json_tool_retry({"model_info": model_info})
|
||||
retry_model_info: Final = _MODEL_INFO.validate_python(retry_params.get("model_info"))
|
||||
return litellm_params.model_copy(update={"model_info": retry_model_info})
|
||||
|
||||
def should_retry_anthropic_messages_without_native_output(
|
||||
self, e: httpx.HTTPStatusError, request_body: Mapping[str, object], model: str
|
||||
|
|
|
|||
|
|
@ -143,7 +143,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
bedrock_route_for_request,
|
||||
bedrock_strict_tools_fallback,
|
||||
deployment_model_info,
|
||||
without_bedrock_native_output_opt_in,
|
||||
for_bedrock_json_tool_retry,
|
||||
without_bedrock_route_prefix,
|
||||
)
|
||||
from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config
|
||||
|
|
@ -4369,7 +4369,7 @@ def _native_output_fallback(
|
|||
first_attempt_params: Mapping[str, object],
|
||||
first_attempt_litellm_params: Mapping[str, object],
|
||||
) -> NativeOutputFallback:
|
||||
litellm_params: Final = without_bedrock_native_output_opt_in(first_attempt_litellm_params)
|
||||
litellm_params: Final = for_bedrock_json_tool_retry(first_attempt_litellm_params)
|
||||
remapped_params: Final = ctx.optional_params_for_model_info(deployment_model_info(litellm_params))
|
||||
return NativeOutputFallback(
|
||||
optional_params={
|
||||
|
|
@ -4500,7 +4500,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu
|
|||
converse_attempt,
|
||||
(
|
||||
partial(_converse_native_output_fallback, ctx, converse_params, dict(litellm_params))
|
||||
if AmazonConverseConfig().uses_opt_in_native_output(converse_model, converse_params, litellm_params)
|
||||
if AmazonConverseConfig().uses_native_output(converse_model, converse_params, litellm_params)
|
||||
else None
|
||||
),
|
||||
converse_model,
|
||||
|
|
@ -4556,7 +4556,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu
|
|||
(
|
||||
partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params))
|
||||
if isinstance(provider_config, AmazonAnthropicClaudeConfig)
|
||||
and provider_config.uses_opt_in_native_output(model, invoke_params, litellm_params)
|
||||
and provider_config.uses_native_output(model, invoke_params, litellm_params)
|
||||
else None
|
||||
),
|
||||
model,
|
||||
|
|
|
|||
|
|
@ -2242,7 +2242,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
@ -2331,7 +2331,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
@ -2420,7 +2420,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
@ -2504,7 +2504,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
@ -2593,7 +2593,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
@ -2715,7 +2715,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_bedrock_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
|
|
@ -2759,7 +2759,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_bedrock_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
|
|
@ -2803,7 +2803,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_bedrock_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
|
|
@ -2847,7 +2847,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_bedrock_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
|
|
@ -2891,7 +2891,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_bedrock_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
|
|
@ -75504,7 +75504,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_bedrock_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
|
|
@ -75544,7 +75544,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
|
|||
|
|
@ -2242,7 +2242,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
@ -2331,7 +2331,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
@ -2420,7 +2420,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
@ -2504,7 +2504,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
@ -2593,7 +2593,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
@ -2715,7 +2715,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_bedrock_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
|
|
@ -2759,7 +2759,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_bedrock_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
|
|
@ -2803,7 +2803,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_bedrock_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
|
|
@ -2847,7 +2847,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_bedrock_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
|
|
@ -2891,7 +2891,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_bedrock_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
|
|
@ -75504,7 +75504,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_bedrock_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
|
|
@ -75544,7 +75544,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
|
|||
|
|
@ -2047,8 +2047,9 @@ def _invoke_chat_params(
|
|||
)
|
||||
@pytest.mark.parametrize("model", ["us.anthropic.claude-opus-4-8", "global.anthropic.claude-sonnet-5"])
|
||||
def test_invoke_uses_opt_in_native_output_when_the_opt_in_drove_native_output(
|
||||
local_model_cost_map, model, thinking_params
|
||||
local_model_cost_map, model, thinking_params, monkeypatch
|
||||
):
|
||||
monkeypatch.setitem(litellm.model_cost[model], "supports_native_structured_output", False)
|
||||
optional_params: Final = _invoke_chat_params(
|
||||
model, None, {"response_format": _CITY_RESPONSE_FORMAT, **thinking_params}
|
||||
)
|
||||
|
|
@ -2083,6 +2084,15 @@ def test_invoke_uses_opt_in_native_output_is_false_when_native_output_did_not_co
|
|||
)
|
||||
|
||||
|
||||
def test_invoke_uses_default_native_output_for_schema_retry(local_model_cost_map: None) -> None:
|
||||
model: Final = "us.anthropic.claude-opus-5"
|
||||
optional_params: Final = _invoke_chat_params(
|
||||
model, None, {"response_format": _CITY_RESPONSE_FORMAT, "thinking": {"type": "disabled"}}
|
||||
)
|
||||
|
||||
assert AmazonAnthropicClaudeConfig().uses_native_output(model, optional_params, {})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("forced_tool_use", "map_flag", "model_info", "thinking_params", "native"),
|
||||
[
|
||||
|
|
|
|||
|
|
@ -5244,6 +5244,44 @@ def test_models_whose_thinking_stops_under_native_output_keep_the_json_tool_only
|
|||
assert ("toolConfig" in request) is not native
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize(
|
||||
("model", "thinking_params", "native"),
|
||||
[
|
||||
("us.anthropic.claude-opus-5", None, True),
|
||||
("us.anthropic.claude-opus-5", {"thinking": {"type": "adaptive"}}, True),
|
||||
("us.anthropic.claude-opus-5", {"thinking": {"type": "disabled"}}, True),
|
||||
("us.anthropic.claude-opus-4-8", None, True),
|
||||
("us.anthropic.claude-opus-4-8", {"thinking": {"type": "adaptive"}}, True),
|
||||
("us.anthropic.claude-opus-4-8", {"thinking": {"type": "disabled"}}, True),
|
||||
("us-gov.anthropic.claude-opus-5", None, False),
|
||||
("us-gov.anthropic.claude-opus-4-8", None, False),
|
||||
],
|
||||
ids=[
|
||||
"opus-5-unset",
|
||||
"opus-5-adaptive",
|
||||
"opus-5-disabled",
|
||||
"opus-4-8-unset",
|
||||
"opus-4-8-adaptive",
|
||||
"opus-4-8-disabled",
|
||||
"opus-5-us-gov",
|
||||
"opus-4-8-us-gov",
|
||||
],
|
||||
)
|
||||
def test_opus_5_and_4_8_cost_map_defaults_use_native_output_across_thinking_modes(
|
||||
model: str, thinking_params: Mapping[str, object] | None, native: bool
|
||||
) -> None:
|
||||
_, request = _converse_request_for_json_schema(model, False, thinking_params)
|
||||
|
||||
if native:
|
||||
assert request["outputConfig"]["textFormat"]["type"] == "json_schema"
|
||||
assert "toolConfig" not in request
|
||||
return
|
||||
|
||||
assert "outputConfig" not in request
|
||||
assert [tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]] == ["json_tool_call"]
|
||||
|
||||
|
||||
def _converse_capital_answer(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
|
|
@ -11599,7 +11637,10 @@ _OUTPUT_CONFIG_FORMAT: Final = {
|
|||
ids=["adaptive", "reasoning_effort", "budget", "output_config_format"],
|
||||
)
|
||||
@pytest.mark.parametrize("model", ["us.anthropic.claude-opus-4-8", "global.anthropic.claude-sonnet-5"])
|
||||
def test_uses_opt_in_native_output_when_the_opt_in_drove_native_output(local_model_cost_map, model, request_params):
|
||||
def test_uses_opt_in_native_output_when_the_opt_in_drove_native_output(
|
||||
local_model_cost_map, monkeypatch, model, request_params
|
||||
):
|
||||
monkeypatch.setitem(litellm.model_cost[model], "supports_native_structured_output", False)
|
||||
optional_params: Final = _converse_params(model, None, request_params)
|
||||
|
||||
assert AmazonConverseConfig().uses_opt_in_native_output(model, optional_params, {"model_info": {"id": "d1"}})
|
||||
|
|
@ -11630,7 +11671,10 @@ def test_uses_opt_in_native_output_is_false_when_native_output_did_not_come_from
|
|||
assert not AmazonConverseConfig().uses_opt_in_native_output(model, optional_params, {"model_info": model_info})
|
||||
|
||||
|
||||
def test_uses_opt_in_native_output_reads_the_base_model_of_an_application_inference_profile(local_model_cost_map):
|
||||
def test_uses_opt_in_native_output_reads_the_base_model_of_an_application_inference_profile(
|
||||
local_model_cost_map, monkeypatch
|
||||
):
|
||||
monkeypatch.setitem(litellm.model_cost["us.anthropic.claude-opus-4-8"], "supports_native_structured_output", False)
|
||||
profile: Final = "arn:aws:bedrock:us-east-1:000000000000:application-inference-profile/profile-id"
|
||||
litellm_params: Final = {"base_model": "bedrock/us.anthropic.claude-opus-4-8"}
|
||||
optional_params: Final = _converse_params(
|
||||
|
|
@ -11698,8 +11742,9 @@ _BEDROCK_SCHEMA_REFUSAL: Final = "output_config.format.schema: pattern is too co
|
|||
)
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_a_refused_native_schema_is_sent_again_only_when_the_retry_drops_native_output(
|
||||
caller_params: dict, expected_calls: int, respx_mock: respx.Router, local_model_cost_map: None
|
||||
caller_params: dict, expected_calls: int, respx_mock: respx.Router, local_model_cost_map: None, monkeypatch
|
||||
) -> None:
|
||||
monkeypatch.setitem(litellm.model_cost["us.anthropic.claude-opus-4-8"], "supports_native_structured_output", False)
|
||||
route: Final = respx_mock.post(url__regex=r"/converse$").mock(
|
||||
return_value=httpx.Response(400, json={"message": _BEDROCK_SCHEMA_REFUSAL})
|
||||
)
|
||||
|
|
@ -11722,6 +11767,61 @@ def test_a_refused_native_schema_is_sent_again_only_when_the_retry_drops_native_
|
|||
assert all("outputConfig" not in json.loads(call.request.content) for call in route.calls[1:])
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_a_default_native_schema_rejection_retries_with_the_json_tool(
|
||||
respx_mock: respx.Router, local_model_cost_map: None
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(url__regex=r"/converse$").mock(
|
||||
return_value=httpx.Response(400, json={"message": _BEDROCK_SCHEMA_REFUSAL})
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=re.escape(_BEDROCK_SCHEMA_REFUSAL)):
|
||||
completion(
|
||||
model="bedrock/converse/us.anthropic.claude-opus-5",
|
||||
messages=[{"role": "user", "content": "What is the capital of France?"}],
|
||||
response_format=CAPITAL_RESPONSE_FORMAT,
|
||||
thinking={"type": "disabled"},
|
||||
num_retries=0,
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-east-1",
|
||||
client=HTTPHandler(),
|
||||
)
|
||||
|
||||
assert route.call_count == 2
|
||||
first_request: Final = json.loads(route.calls[0].request.content)
|
||||
retry_request: Final = json.loads(route.calls[1].request.content)
|
||||
assert "outputConfig" in first_request and "toolConfig" not in first_request
|
||||
assert "toolConfig" in retry_request and "outputConfig" not in retry_request
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_an_unforceable_tool_native_schema_rejection_retries_with_the_json_tool(
|
||||
respx_mock: respx.Router, local_model_cost_map: None
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(url__regex=r"/converse$").mock(
|
||||
return_value=httpx.Response(400, json={"message": _BEDROCK_SCHEMA_REFUSAL})
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=re.escape(_BEDROCK_SCHEMA_REFUSAL)):
|
||||
completion(
|
||||
model="bedrock/converse/anthropic.claude-fable-5-1",
|
||||
messages=[{"role": "user", "content": "What is the capital of France?"}],
|
||||
response_format=CAPITAL_RESPONSE_FORMAT,
|
||||
num_retries=0,
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-east-1",
|
||||
client=HTTPHandler(),
|
||||
)
|
||||
|
||||
assert route.call_count == 2
|
||||
first_request: Final = json.loads(route.calls[0].request.content)
|
||||
retry_request: Final = json.loads(route.calls[1].request.content)
|
||||
assert "outputConfig" in first_request and "toolConfig" not in first_request
|
||||
assert "toolConfig" in retry_request and "outputConfig" not in retry_request
|
||||
|
||||
|
||||
CONTROL_PLANE_ENDPOINT: Final = "https://bedrock-fips.us-east-1.amazonaws.com"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1214,7 +1214,7 @@ async def test_converse_json_schema_stream_sends_claude_prose_only_when_claude_s
|
|||
|
||||
transport: Final = httpx.MockTransport(handle_request)
|
||||
request: Final = {
|
||||
"model": "bedrock/us.anthropic.claude-opus-5",
|
||||
"model": "bedrock/anthropic.claude-opus-5",
|
||||
"messages": [{"role": "user", "content": "What is the LCM of 24 and 120? Reply in JSON."}],
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
|
|
|
|||
|
|
@ -13,10 +13,33 @@ _MODEL: Final = "us.anthropic.claude-opus-4-8"
|
|||
_FALLBACK: Final = NativeOutputFallback(
|
||||
optional_params={"tools": "json_tool_call"}, litellm_params={"model_info": {"opted_out": True}}
|
||||
)
|
||||
_NATIVE_RETRY_WITH_STRICT_TOOLS: Final = NativeOutputFallback(
|
||||
optional_params={
|
||||
"tools": (
|
||||
{"function": {"name": "json_schema", "strict": True}},
|
||||
{"function": {"name": "custom", "strict": True}},
|
||||
)
|
||||
},
|
||||
litellm_params={
|
||||
"model_info": {
|
||||
"supports_native_structured_output": False,
|
||||
"supports_bedrock_native_structured_output": False,
|
||||
}
|
||||
},
|
||||
)
|
||||
_COMPOSED_FALLBACK: Final = NativeOutputFallback(
|
||||
optional_params={"tools": [{"function": {"name": "json_schema"}}, {"function": {"name": "custom"}}]},
|
||||
litellm_params=_NATIVE_RETRY_WITH_STRICT_TOOLS.litellm_params,
|
||||
)
|
||||
_REJECTION: Final = BedrockError(
|
||||
status_code=400,
|
||||
message='{"message":"The model returned the following errors: output_config.format: Extra inputs are not permitted"}',
|
||||
)
|
||||
_GRAMMAR_REJECTION: Final = BedrockError(
|
||||
status_code=400,
|
||||
message='{"message":"The model returned the following errors: The compiled grammar is too large, which would cause '
|
||||
'performance issues. Simplify your tool schemas or reduce the number of strict tools."}',
|
||||
)
|
||||
_SCHEMA_REFUSAL: Final = BedrockError(
|
||||
status_code=400, message='{"message":"output_config.format.schema: pattern is too complex for structured output"}'
|
||||
)
|
||||
|
|
@ -58,8 +81,15 @@ _MODES: Final = pytest.mark.parametrize("asynchronous", [False, True], ids=["syn
|
|||
|
||||
|
||||
@_MODES
|
||||
def test_a_native_output_rejection_is_sent_again_once_with_the_fallback_params(asynchronous: bool) -> None:
|
||||
bedrock: Final = ScriptedBedrock(errors=(_REJECTION, None), asynchronous=asynchronous)
|
||||
@pytest.mark.parametrize(
|
||||
"rejection",
|
||||
[_REJECTION, _GRAMMAR_REJECTION],
|
||||
ids=["format_not_permitted", "compiled_grammar_too_large"],
|
||||
)
|
||||
def test_a_native_output_rejection_is_sent_again_once_with_the_fallback_params(
|
||||
asynchronous: bool, rejection: BedrockError
|
||||
) -> None:
|
||||
bedrock: Final = ScriptedBedrock(errors=(rejection, None), asynchronous=asynchronous)
|
||||
|
||||
response: Final = _call(bedrock, _FALLBACK)
|
||||
|
||||
|
|
@ -77,6 +107,19 @@ def test_a_second_native_output_rejection_reaches_the_caller(asynchronous: bool)
|
|||
assert len(bedrock.sent) == 2
|
||||
|
||||
|
||||
@_MODES
|
||||
def test_a_grammar_rejection_after_a_native_retry_reaches_the_caller_without_strict_tools_fallback(
|
||||
asynchronous: bool,
|
||||
) -> None:
|
||||
bedrock: Final = ScriptedBedrock(errors=(_GRAMMAR_REJECTION, _GRAMMAR_REJECTION), asynchronous=asynchronous)
|
||||
|
||||
with pytest.raises(BedrockError) as raised:
|
||||
_resolve(call_with_native_output_fallback(bedrock, lambda: _FALLBACK, _MODEL))
|
||||
|
||||
assert raised.value is _GRAMMAR_REJECTION
|
||||
assert bedrock.sent == (None, _FALLBACK)
|
||||
|
||||
|
||||
@_MODES
|
||||
@pytest.mark.parametrize(
|
||||
("error", "fallback"),
|
||||
|
|
@ -155,13 +198,16 @@ _OTHER_400S_WITH_A_STRICT_TOOL: Final = tuple(
|
|||
|
||||
|
||||
def _call_with_strict_tools_fallback(
|
||||
bedrock: ScriptedBedrock, fallback: NativeOutputFallback | None, strict_tools_fallback: NativeOutputFallback | None
|
||||
bedrock: ScriptedBedrock,
|
||||
fallback: NativeOutputFallback | None,
|
||||
strict_tools_fallback: NativeOutputFallback | None,
|
||||
model: str = _MODEL,
|
||||
) -> object:
|
||||
return _resolve(
|
||||
call_with_native_output_fallback(
|
||||
bedrock,
|
||||
None if fallback is None else lambda: fallback,
|
||||
_MODEL,
|
||||
model,
|
||||
strict_tools_fallback=None if strict_tools_fallback is None else lambda: strict_tools_fallback,
|
||||
)
|
||||
)
|
||||
|
|
@ -223,3 +269,18 @@ def test_each_rejection_is_sent_again_with_its_own_fallback_when_both_are_set(
|
|||
_call_with_strict_tools_fallback(bedrock, _FALLBACK, _STRICT_FALLBACK)
|
||||
|
||||
assert bedrock.sent == (None, retried_with)
|
||||
|
||||
|
||||
@_MODES
|
||||
def test_a_strict_tools_rejection_after_native_retry_composes_both_fallbacks(asynchronous: bool) -> None:
|
||||
bedrock: Final = ScriptedBedrock(
|
||||
errors=(_GRAMMAR_REJECTION, _GRAMMAR_REJECTION, None),
|
||||
asynchronous=asynchronous,
|
||||
)
|
||||
|
||||
response: Final = _call_with_strict_tools_fallback(
|
||||
bedrock, _NATIVE_RETRY_WITH_STRICT_TOOLS, _STRICT_FALLBACK, "us.anthropic.claude-sonnet-4-6"
|
||||
)
|
||||
|
||||
assert isinstance(response, ModelResponse) and response.id == "answer-3"
|
||||
assert bedrock.sent == (None, _NATIVE_RETRY_WITH_STRICT_TOOLS, _COMPOSED_FALLBACK)
|
||||
|
|
|
|||
|
|
@ -2047,7 +2047,7 @@ class TestBedrockBatchAnthropicRowParams:
|
|||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "native_output"),
|
||||
[("bedrock/us.anthropic.claude-opus-4-8", False), (MODEL, True)],
|
||||
[("bedrock/us.anthropic.claude-sonnet-5-5", False), (MODEL, True)],
|
||||
ids=["opt_in_native_output_model", "native_output_model"],
|
||||
)
|
||||
def test_thinking_json_schema_rows_use_native_output_only_on_models_that_always_take_it(
|
||||
|
|
|
|||
|
|
@ -4016,7 +4016,10 @@ _NATIVE_OUTPUT_REJECTION: Final = {
|
|||
"message": "The model returned the following errors: compilation exceeded limits: Schema is too complex."
|
||||
}
|
||||
_STALE_SIGNATURE_ERROR: Final = {"message": "messages.1.content.0: Invalid `signature` in `thinking` block"}
|
||||
_OPTED_OUT: Final = {"supports_bedrock_native_structured_output": False}
|
||||
_OPTED_OUT: Final = {
|
||||
"supports_native_structured_output": False,
|
||||
"supports_bedrock_native_structured_output": False,
|
||||
}
|
||||
_CONSTRAINED_SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
|
@ -4158,21 +4161,21 @@ async def test_bedrock_messages_native_output_rejection_and_signature_400_each_g
|
|||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize(
|
||||
("model", "model_info", "error"),
|
||||
("model", "model_info", "error", "should_retry"),
|
||||
[
|
||||
("us.anthropic.claude-opus-4-8", None, {"message": "messages.0.content: Field required"}),
|
||||
("us.anthropic.claude-sonnet-4-6", None, _NATIVE_OUTPUT_REJECTION),
|
||||
("us.anthropic.claude-opus-4-8", _OPTED_OUT, _NATIVE_OUTPUT_REJECTION),
|
||||
("us.anthropic.claude-opus-4-8", None, {"message": "messages.0.content: Field required"}, False),
|
||||
("us.anthropic.claude-sonnet-4-6", None, _NATIVE_OUTPUT_REJECTION, True),
|
||||
("us.anthropic.claude-opus-4-8", _OPTED_OUT, _NATIVE_OUTPUT_REJECTION, False),
|
||||
],
|
||||
ids=["other_400", "native_model", "deployment_opt_out"],
|
||||
ids=["other_400", "native_rejection", "deployment_opt_out"],
|
||||
)
|
||||
async def test_bedrock_messages_400_is_not_retried_unless_it_rejects_opt_in_native_output(
|
||||
model: str, model_info: Mapping[str, object] | None, error: Mapping[str, object]
|
||||
async def test_bedrock_messages_retries_only_native_output_rejections(
|
||||
model: str, model_info: Mapping[str, object] | None, error: Mapping[str, object], should_retry: bool
|
||||
) -> None:
|
||||
bedrock, raised = await _send_bedrock_messages(model, model_info, [(400, error), (200, _ANTHROPIC_MESSAGE)])
|
||||
|
||||
assert isinstance(raised, litellm.BadRequestError)
|
||||
assert len(bedrock.bodies) == 1
|
||||
assert (raised is None) is should_retry
|
||||
assert len(bedrock.bodies) == (2 if should_retry else 1)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from litellm.llms.bedrock.common_utils import(
|
|||
bedrock_strict_tools_fallback,
|
||||
deployment_model_info,
|
||||
extract_model_name_from_bedrock_arn,
|
||||
for_bedrock_json_tool_retry,
|
||||
get_bedrock_base_model,
|
||||
get_bedrock_cross_region_inference_regions,
|
||||
is_bedrock_native_output_rejection,
|
||||
|
|
@ -2328,6 +2329,10 @@ def test_bedrock_capability_model_reads_base_model_only_for_application_profile_
|
|||
|
||||
_FORMAT_REJECTION: Final = "The model returned the following errors: output_config.format: Extra inputs are not permitted"
|
||||
_COMPILE_REJECTION: Final = "The model returned the following errors: compilation exceeded limits: Schema is too complex."
|
||||
_GRAMMAR_REJECTION: Final = (
|
||||
"The model returned the following errors: The compiled grammar is too large, which would cause performance issues. "
|
||||
"Simplify your tool schemas or reduce the number of strict tools."
|
||||
)
|
||||
_SCHEMA_PATTERN_REJECTION: Final = (
|
||||
"The model returned the following errors: output_config.format.schema: Unsupported regex feature in pattern "
|
||||
"field: pattern is too complex for structured output: reduce the {n,m} upper bound, narrow the character class "
|
||||
|
|
@ -2340,6 +2345,7 @@ _SCHEMA_PATTERN_REJECTION: Final = (
|
|||
[
|
||||
(400, f'{{"message":"{_FORMAT_REJECTION}"}}', True),
|
||||
(400, f'{{"message":"{_COMPILE_REJECTION}"}}', True),
|
||||
(400, _GRAMMAR_REJECTION, True),
|
||||
(400, f'{{"message":"{_SCHEMA_PATTERN_REJECTION}"}}', True),
|
||||
(400, "Malformed input request: #: extraneous key [output_config] is not permitted", False),
|
||||
(400, "The model returned the following errors: messages.0.content: Field required", False),
|
||||
|
|
@ -2349,6 +2355,7 @@ _SCHEMA_PATTERN_REJECTION: Final = (
|
|||
ids=[
|
||||
"format_not_permitted",
|
||||
"schema_too_complex",
|
||||
"compiled_grammar_too_large",
|
||||
"schema_pattern_too_complex",
|
||||
"other_key_not_permitted",
|
||||
"other_400",
|
||||
|
|
@ -2365,7 +2372,7 @@ def test_is_bedrock_native_output_rejection_matches_only_the_400s_that_reject_na
|
|||
@pytest.mark.parametrize(
|
||||
("model", "model_info", "expected"),
|
||||
[
|
||||
("us.anthropic.claude-opus-4-8", None, True),
|
||||
("us.anthropic.claude-opus-4-8", None, False),
|
||||
("global.anthropic.claude-sonnet-5", None, True),
|
||||
("us-gov.anthropic.claude-opus-4-8", None, False),
|
||||
("us.anthropic.claude-opus-4-8", {"supports_bedrock_native_structured_output": False}, False),
|
||||
|
|
@ -2389,9 +2396,43 @@ def test_without_bedrock_native_output_opt_in_turns_the_opt_in_off_and_keeps_the
|
|||
"us.anthropic.claude-opus-4-8", model_info=deployment_model_info(turned_off)
|
||||
)
|
||||
assert turned_off["api_base"] == "https://bedrock.example"
|
||||
assert deployment_model_info(turned_off)["id"] == "deployment-1"
|
||||
assert deployment_model_info(turned_off) == {
|
||||
"id": "deployment-1",
|
||||
"supports_bedrock_native_structured_output": False,
|
||||
}
|
||||
assert litellm_params["model_info"] == {"id": "deployment-1"}
|
||||
|
||||
native_default: Final = without_bedrock_native_output_opt_in(
|
||||
{"model_info": {"supports_native_structured_output": True}}
|
||||
)
|
||||
assert deployment_model_info(native_default) == {
|
||||
"supports_native_structured_output": True,
|
||||
"supports_bedrock_native_structured_output": False,
|
||||
}
|
||||
|
||||
|
||||
def test_for_bedrock_json_tool_retry_disables_both_native_output_flags(local_model_cost_map: None) -> None:
|
||||
litellm_params: Final = {
|
||||
"model_info": {
|
||||
"id": "deployment-1",
|
||||
"supports_native_structured_output": True,
|
||||
"supports_bedrock_native_structured_output": True,
|
||||
}
|
||||
}
|
||||
|
||||
retry_params: Final = for_bedrock_json_tool_retry(litellm_params)
|
||||
|
||||
assert deployment_model_info(retry_params) == {
|
||||
"id": "deployment-1",
|
||||
"supports_native_structured_output": False,
|
||||
"supports_bedrock_native_structured_output": False,
|
||||
}
|
||||
assert litellm_params["model_info"] == {
|
||||
"id": "deployment-1",
|
||||
"supports_native_structured_output": True,
|
||||
"supports_bedrock_native_structured_output": True,
|
||||
}
|
||||
|
||||
|
||||
def test_bedrock_response_stream_shape_is_unavailable_when_botocore_describes_the_service_as_a_list():
|
||||
import litellm.llms.bedrock.common_utils as mod
|
||||
|
|
|
|||
|
|
@ -332,6 +332,18 @@ def _cross_region_profiles(
|
|||
)
|
||||
|
||||
|
||||
_BEDROCK_PROFILE_ONLY_NATIVE_OUTPUT_MODELS: Final = frozenset(
|
||||
{
|
||||
"anthropic.claude-opus-5-5",
|
||||
"anthropic.claude-fable-5-1",
|
||||
"anthropic.claude-sonnet-5",
|
||||
"anthropic.claude-haiku-5-5",
|
||||
"anthropic.claude-opus-5",
|
||||
"anthropic.claude-opus-4-8",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_get_model_info_bedrock_cross_region_capability_parity():
|
||||
"""
|
||||
Cross-region inference profiles carry litellm_provider "bedrock_converse", so the
|
||||
|
|
@ -362,6 +374,13 @@ def test_get_model_info_bedrock_cross_region_capability_parity():
|
|||
if _govcloud_may_lag(k, cap):
|
||||
assert base_value or not v[cap], f"{cap} is on {k} but not on {base_model_key}"
|
||||
continue
|
||||
if (
|
||||
cap == "supports_native_structured_output"
|
||||
and base_model_key in _BEDROCK_PROFILE_ONLY_NATIVE_OUTPUT_MODELS
|
||||
and v[cap] is True
|
||||
):
|
||||
assert base_value is False, f"{cap} is already enabled on bare model {base_model_key}"
|
||||
continue
|
||||
assert (
|
||||
v[cap] == base_value
|
||||
), f"{cap} is {v[cap]} on {k} but {base_value} on {base_model_key}"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue