From a9bad2b5e0178b5bb5b06e5764f62643e03aedc5 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 18 Oct 2026 00:06:07 +0000 Subject: [PATCH] fix(bedrock): send native structured output first on Opus 4.8 and Sonnet 5 and fall back to the json tool when Bedrock rejects it (#47100) * fix(bedrock): send native structured output first on Opus 4.8 and Sonnet 5 and fall back to the json tool when Bedrock rejects it * fix(bedrock): retry with the json tool when Bedrock refuses the native output schema Count a 400 under output_config.format.schema: as a native output rejection, so a schema Bedrock cannot compile (like a pattern with a nested {n,m} group) gets the json tool retry instead of reaching the caller * fix(bedrock): make the native output retry send what an opted-out deployment sends The retry after a native output rejection now reruns the same optional param mapping completion() ran, with the deployment model_info opted out of supports_bedrock_native_structured_output, so tool_choice, effort and the json tool schema match an opted-out deployment instead of a hand-built tool. Each config keeps only the predicate that decides when native output came from the opt-in key. Bedrock batch records for Anthropic models map with the opt-in turned off, since a batch job has no retry. Models on supports_native_structured_output keep native output in batch * fix(bedrock): build the messages retry from the caller's request and keep other errors unchanged * fix(bedrock): give sync converse stream errors the upstream body as text * fix(bedrock): let a malformed caller outputConfig reach Bedrock instead of raising A Converse json_schema request whose caller outputConfig failed litellm's shape check raised a pydantic error before the request was sent, so the caller got a 500 where the merge base passed Bedrock's 400 through. The opt-in check now treats that outputConfig as carrying no native output. Also opts the vertex wire test's pinned Opus 4.8 json tool deployment out of native output, and extends the Bedrock native output integration tests. * fix(bedrock): send a refused caller outputConfig once and give opt-out advice only when native output is unsupported A caller can send its own outputConfig on Converse. When Bedrock refused that schema on an opt-in deployment, the retry still carried the caller's textFormat, so Bedrock got the same request twice and the warning told the operator to turn off an opt-in that was not the cause. The retry is now skipped when the opted-out request would still carry a native textFormat, so Bedrock's 400 reaches the caller after one call as on the merge base. The opt-out advice is logged only for the account-level "Extra inputs are not permitted" rejection. * fix(bedrock): keep the caller's continuation message on the native output retry The Converse transform pops user_continue_message from litellm_params on the first attempt, so a retry built from those params afterwards dropped it. Both Bedrock chat routes now hand the fallback a copy taken before the first attempt. * test(bedrock): pin additional_drop_params on the /v1/messages native output retry * fix(bedrock): give the Converse native output retry the fake stream timeout --- .../anthropic_messages/transformation.py | 21 +- litellm/llms/bedrock/chat/converse_handler.py | 24 +- .../bedrock/chat/converse_transformation.py | 32 +- .../anthropic_claude3_transformation.py | 6 + .../bedrock/chat/native_output_fallback.py | 62 +++ litellm/llms/bedrock/common_utils.py | 81 +++- litellm/llms/bedrock/files/transformation.py | 6 +- .../anthropic_claude3_transformation.py | 27 ++ litellm/llms/custom_httpx/llm_http_handler.py | 87 ++-- litellm/main.py | 148 +++++-- litellm/types/completion.py | 1 + .../test_bedrock_batch_files_wire.py | 55 ++- ...t_bedrock_native_structured_output_wire.py | 390 +++++++++++++++++- ...vertex_anthropic_structured_output_wire.py | 9 +- ...drock_converse_stream_sync_decoder_wire.py | 14 + ...test_bedrock_native_output_fallback_sdk.py | 336 +++++++++++++++ ...ations_anthropic_claude3_transformation.py | 63 +++ .../chat/test_bedrock_converse_handler.py | 46 +++ .../chat/test_converse_transformation.py | 153 +++++++ .../chat/test_native_output_fallback.py | 123 ++++++ .../test_bedrock_files_transformation.py | 25 ++ .../test_anthropic_claude3_transformation.py | 160 +++++++ .../llms/bedrock/test_bedrock_common_utils.py | 77 ++++ tests/unit/types/test_completion.py | 1 + 24 files changed, 1859 insertions(+), 88 deletions(-) create mode 100644 litellm/llms/bedrock/chat/native_output_fallback.py create mode 100644 tests/integration/sdk/test_bedrock_native_output_fallback_sdk.py create mode 100644 tests/unit/llms/bedrock/chat/test_native_output_fallback.py diff --git a/litellm/llms/base_llm/anthropic_messages/transformation.py b/litellm/llms/base_llm/anthropic_messages/transformation.py index bd6c53a0453..06717207dc0 100644 --- a/litellm/llms/base_llm/anthropic_messages/transformation.py +++ b/litellm/llms/base_llm/anthropic_messages/transformation.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Mapping from typing import TYPE_CHECKING, Any, Final import httpx @@ -180,7 +180,7 @@ class BaseAnthropicMessagesConfig(ABC): return 2 def should_retry_anthropic_messages_on_http_error( - self, e: httpx.HTTPStatusError, litellm_params: dict, request_body: object = None + self, e: httpx.HTTPStatusError, litellm_params: Mapping[str, object], request_body: object = None ) -> bool: """ When True, async_anthropic_messages_handler will transform the request body @@ -202,7 +202,9 @@ class BaseAnthropicMessagesConfig(ABC): ) and is_anthropic_thinking_binding_mismatch_error(error_text) return not caller_asked_for_this_error and is_anthropic_invalid_thinking_block_error(error_text) - def transform_anthropic_messages_request_on_http_error(self, e: httpx.HTTPStatusError, request_data: dict) -> dict: + def transform_anthropic_messages_request_on_http_error( + self, e: httpx.HTTPStatusError, request_data: dict[str, object] + ) -> dict[str, object]: """ Mutates request_data in place when retrying after a recoverable HTTP error. """ @@ -214,3 +216,16 @@ class BaseAnthropicMessagesConfig(ABC): if e.response.status_code == 400 and is_anthropic_invalid_thinking_block_error(e.response.text): strip_thinking_blocks_from_anthropic_messages_request_dict(request_data) return request_data + + def native_output_fallback_litellm_params( + self, + model: str, + anthropic_messages_optional_request_params: Mapping[str, object], + litellm_params: GenericLiteLLMParams, + ) -> GenericLiteLLMParams | None: + return None + + def should_retry_anthropic_messages_without_native_output( + self, e: httpx.HTTPStatusError, request_body: Mapping[str, object], model: str + ) -> bool: + return False diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index 021327eaa1f..d8f7c5bc3b6 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -39,14 +39,22 @@ def make_sync_call( if client is None: client = get_httpx_client() # Create a new client if none provided - response: Final = client.post( - api_base, - headers=headers, - data=data, - stream=not fake_stream, - logging_obj=logging_obj, - timeout=timeout, - ) + try: + response: Final = client.post( + api_base, + headers=headers, + data=data, + stream=not fake_stream, + logging_obj=logging_obj, + timeout=timeout, + ) + except httpx.HTTPStatusError as err: + raise BedrockError( + status_code=err.response.status_code, + message=error_response_text(err.response), + headers=err.response.headers, + response=err.response, + ) from err if response.status_code != 200: raise BedrockError( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 6c563084031..954bacce05d 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -12,7 +12,7 @@ from itertools import chain from typing import TYPE_CHECKING, Final, Literal, cast, overload import httpx -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError import litellm from litellm._logging import verbose_logger @@ -58,7 +58,9 @@ from litellm.llms.bedrock.common_utils import ( bedrock_cleaned_output_schema, bedrock_model_supports_regex_lookaround, bedrock_native_output_format, + bedrock_native_output_is_opt_in, deployment_model_info, + requests_output_config_format, ) from litellm.llms.bedrock.request_metadata import ( bedrock_request_metadata_headers, @@ -151,6 +153,18 @@ def _tools_the_model_accepts( return [tool_with_sanitized_parameters(tool, drop_lookaround_regex_patterns) for tool in tools] +_OUTPUT_CONFIG_BLOCK: Final = TypeAdapter(OutputConfigBlock) + + +def _has_native_text_format(output_config: object) -> bool: + if not isinstance(output_config, Mapping): + return False + try: + return _OUTPUT_CONFIG_BLOCK.validate_python(output_config).get("textFormat") is not None + except ValidationError: + return False + + class AmazonConverseConfig(BaseConfig): """ Reference - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html @@ -1358,6 +1372,22 @@ class AmazonConverseConfig(BaseConfig): optional_params["json_mode"] = True return optional_params + @staticmethod + def sends_native_text_format(optional_params: Mapping[str, object]) -> bool: + return _has_native_text_format(optional_params.get("outputConfig")) + + def uses_opt_in_native_output( + self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object] + ) -> bool: + native_output_requested: Final = ( + optional_params.get("json_mode") is True and self.sends_native_text_format(optional_params) + ) or requests_output_config_format(optional_params) + return native_output_requested and bedrock_native_output_is_opt_in( + bedrock_capability_model(model, litellm_params.get("base_model")), + self.custom_llm_provider or "bedrock_converse", + deployment_model_info(litellm_params), + ) + def update_optional_params_with_thinking_tokens(self, non_default_params: dict, optional_params: dict): """ Handles scenario where max tokens is not specified. For anthropic models (anthropic api/bedrock/vertex ai), this requires having the max tokens being set and being greater than the thinking token budget. diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 6715cd88068..e7783e01b96 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -26,6 +26,7 @@ from litellm.llms.bedrock.common_utils import ( normalize_bedrock_opus_output_config_effort, normalize_custom_field_on_tools, normalize_tool_input_schema_types_for_bedrock_invoke, + requests_opt_in_native_output, strip_unsupported_bedrock_invoke_output_config_keys, tools_without_eager_input_streaming, ) @@ -99,6 +100,11 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): model, self.is_thinking_requested(request_params), model_info=self._deployment_model_info ) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None) + def uses_opt_in_native_output( + self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object] + ) -> bool: + return requests_opt_in_native_output(model, optional_params, deployment_model_info(litellm_params)) + def map_openai_params( self, non_default_params: dict, diff --git a/litellm/llms/bedrock/chat/native_output_fallback.py b/litellm/llms/bedrock/chat/native_output_fallback.py new file mode 100644 index 00000000000..1b8ce5fcda4 --- /dev/null +++ b/litellm/llms/bedrock/chat/native_output_fallback.py @@ -0,0 +1,62 @@ +from collections.abc import Callable, Coroutine +from typing import Final, TypeAlias + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.bedrock.common_utils import ( + NativeOutputFallback, + is_bedrock_native_output_rejection, + warn_bedrock_native_output_rejected, +) +from litellm.types.utils import ModelResponse +from litellm.utils import CustomStreamWrapper + +NativeOutputResponse: TypeAlias = ModelResponse | CustomStreamWrapper +NativeOutputAttemptResult: TypeAlias = NativeOutputResponse | Coroutine[object, object, NativeOutputResponse] +NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeOutputAttemptResult] +NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None] + + +def _fallback_for( + error: BaseLLMException, fallback: NativeOutputFallbackFactory, model: str +) -> NativeOutputFallback | None: + if not is_bedrock_native_output_rejection(error.status_code, error.message): + return None + retry: Final = fallback() + if retry is not None: + warn_bedrock_native_output_rejected(model, error.message) + return retry + + +async def _await_with_native_output_fallback( + first_attempt: Coroutine[object, object, NativeOutputResponse], + attempt: NativeOutputAttempt, + fallback: NativeOutputFallbackFactory, + model: str, +) -> NativeOutputResponse: + try: + return await first_attempt + except BaseLLMException as error: + retry: Final = _fallback_for(error, fallback, model) + if retry is None: + raise + second_attempt: Final = attempt(retry) + return await second_attempt if isinstance(second_attempt, Coroutine) else second_attempt + + +def call_with_native_output_fallback( + attempt: NativeOutputAttempt, + fallback: NativeOutputFallbackFactory | None, + model: str, +) -> NativeOutputAttemptResult: + if fallback is None: + return attempt(None) + try: + first_attempt: Final = attempt(None) + except BaseLLMException as error: + retry: Final = _fallback_for(error, fallback, model) + if retry is None: + raise + return attempt(retry) + if isinstance(first_attempt, Coroutine): + return _await_with_native_output_fallback(first_attempt, attempt, fallback, model) + return first_attempt diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 4404323f08b..207673301f8 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -10,6 +10,7 @@ import json import os import re from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias @@ -181,7 +182,7 @@ def _get_local_model_cost_map() -> dict: return GetModelCostMap.load_local_model_cost_map() -def pop_bedrock_invoke_output_config_format(request_body: dict) -> dict | None: +def pop_bedrock_invoke_output_config_format(request_body: dict[str, object]) -> dict[str, object] | None: """ Remove and return Anthropic's nested ``output_config.format`` field. @@ -202,8 +203,8 @@ def pop_bedrock_invoke_output_config_format(request_body: dict) -> dict | None: def convert_bedrock_invoke_output_format_to_inline_schema( - output_format: dict, - request_body: dict, + output_format: Mapping[str, object], + request_body: dict[str, object], ) -> None: """ Embed an Anthropic structured-output schema into the last user message. @@ -289,6 +290,77 @@ def bedrock_chat_uses_native_structured_output( ) +def bedrock_native_output_is_opt_in( + model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None +) -> bool: + return not _bedrock_model_supports( + model, "supports_native_structured_output", custom_llm_provider + ) and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info) + + +def without_bedrock_native_output_opt_in(litellm_params: Mapping[str, object]) -> Mapping[str, object]: + return { + **litellm_params, + "model_info": {**deployment_model_info(litellm_params), "supports_bedrock_native_structured_output": False}, + } + + +_BEDROCK_NATIVE_OUTPUT_UNSUPPORTED: Final = "output_config.format: Extra inputs are not permitted" +_BEDROCK_NATIVE_OUTPUT_REJECTIONS: Final = ( + _BEDROCK_NATIVE_OUTPUT_UNSUPPORTED, + "compilation exceeded limits: Schema is too complex", + "output_config.format.schema:", +) + + +def is_bedrock_native_output_rejection(status_code: int, error_text: str) -> bool: + return status_code == 400 and any(rejection in error_text for rejection in _BEDROCK_NATIVE_OUTPUT_REJECTIONS) + + +def warn_bedrock_native_output_rejected(model: str, error_text: str) -> None: + opt_out_advice: Final = ( + " Set `supports_bedrock_native_structured_output: false` in this deployment's model_info to skip the " + "rejected call." + if _BEDROCK_NATIVE_OUTPUT_UNSUPPORTED in error_text + else "" + ) + verbose_logger.warning( + "Bedrock rejected native structured output for model=%s, so litellm sent the request again without it.%s", + model, + opt_out_advice, + ) + + +@dataclass(frozen=True, slots=True) +class NativeOutputFallback: + optional_params: Mapping[str, object] + litellm_params: Mapping[str, object] + + +def requests_output_config_format(optional_params: Mapping[str, object]) -> bool: + match optional_params: + case {"output_config": {"format": object()}}: + return True + case _: + return False + + +def requests_opt_in_native_output( + model: str, anthropic_optional_params: Mapping[str, object], model_info: Mapping[str, object] | None +) -> bool: + native_output_requested: Final = isinstance( + anthropic_optional_params.get("output_format"), Mapping + ) or requests_output_config_format(anthropic_optional_params) + return native_output_requested and bedrock_native_output_is_opt_in(model, model_info=model_info) + + +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) + + +def json_object_or_none(value: object) -> Mapping[str, object] | None: + return _JSON_OBJECT.validate_python(value) if isinstance(value, Mapping) else None + + BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES: Final = 64 @@ -355,7 +427,8 @@ def apply_bedrock_invoke_structured_output( """ legacy_output_format: Final = request_body.pop("output_format", None) output_config_format: Final = pop_bedrock_invoke_output_config_format(request_body) - schema_format: Final = legacy_output_format if isinstance(legacy_output_format, dict) else output_config_format + legacy_format: Final = json_object_or_none(legacy_output_format) + schema_format: Final = legacy_format if legacy_format is not None else output_config_format if schema_format is None: return diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 5c5d6e73417..38da5c3d4f7 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -66,8 +66,10 @@ from litellm.utils import get_llm_provider, get_optional_params from ..base_aws_llm import BaseAWSLLM from ..common_utils import ( BedrockError, + deployment_model_info, merge_bedrock_aws_request_params, resolve_s3_encryption_key_id, + without_bedrock_native_output_opt_in, ) S3_SIGNED_REQUEST_HEADERS_PARAM: Final = "_s3_signed_request_headers" @@ -982,10 +984,12 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): ) config: Final = AmazonAnthropicClaudeConfig() + litellm_params: Final = without_bedrock_native_output_opt_in({}) mapped_params = get_optional_params( model=_invoke_route_model(model), custom_llm_provider="bedrock", messages=messages, + model_info=deployment_model_info(litellm_params), **MappingProxyType( { k: v @@ -1000,7 +1004,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): optional_params={ k: v for k, v in mapped_params.items() if k not in _MAPPED_PARAMS_THE_REQUEST_HANDLER_STRIPS }, - litellm_params={}, + litellm_params=dict(litellm_params), headers={}, ) diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 93e5108dabf..3da80dd83ec 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -40,11 +40,15 @@ from litellm.llms.bedrock.common_utils import ( ensure_bedrock_anthropic_messages_tool_names, get_anthropic_beta_from_headers, is_claude_4_5_on_bedrock, + is_bedrock_native_output_rejection, normalize_bedrock_opus_output_config_effort, normalize_custom_field_on_tools, normalize_tool_input_schema_types_for_bedrock_invoke, + requests_opt_in_native_output, + requests_output_config_format, strip_unsupported_bedrock_invoke_output_config_keys, tools_without_eager_input_streaming, + warn_bedrock_native_output_rejected, ) from litellm.llms.bedrock.request_metadata import ( bedrock_request_metadata_headers, @@ -111,6 +115,29 @@ class AmazonAnthropicClaudeMessagesConfig( BaseAnthropicMessagesConfig.__init__(self, **kwargs) AmazonInvokeConfig.__init__(self, **kwargs) + def native_output_fallback_litellm_params( + self, + model: str, + anthropic_messages_optional_request_params: Mapping[str, object], + litellm_params: GenericLiteLLMParams, + ) -> GenericLiteLLMParams | None: + model_info: Final = _MODEL_INFO.validate_python(litellm_params.model_info) + if not requests_opt_in_native_output(model, anthropic_messages_optional_request_params, model_info): + return None + return litellm_params.model_copy( + update={"model_info": {**(model_info or {}), "supports_bedrock_native_structured_output": False}} + ) + + def should_retry_anthropic_messages_without_native_output( + self, e: httpx.HTTPStatusError, request_body: Mapping[str, object], model: str + ) -> bool: + if not requests_output_config_format(request_body) or not is_bedrock_native_output_rejection( + e.response.status_code, e.response.text + ): + return False + warn_bedrock_native_output_rejected(model, e.response.text) + return True + def validate_anthropic_messages_environment( self, headers: dict, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 9cfc47761dd..d6e862a8139 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1,8 +1,9 @@ import asyncio +import copy import inspect import json import ssl -from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Coroutine, Iterator, Mapping, Sequence +from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence from contextlib import asynccontextmanager from functools import lru_cache, partial from types import MappingProxyType @@ -464,6 +465,9 @@ async def _aiter_bytes_then_close(response: httpx.Response, *, chunk_size: int) _DECODED_BODY_STALE_HEADERS: Final[frozenset[str]] = frozenset({"content-encoding", "content-length"}) +_OPTIONAL_PARAMS: Final = TypeAdapter(Mapping[str, object]) +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) +_JSON_OBJECTS: Final = TypeAdapter(list[dict[str, object]]) def _decoded_body_headers(response: httpx.Response) -> httpx.Headers: @@ -2099,11 +2103,13 @@ class BaseLLMHTTPHandler: api_key: str | None, model: str, timeout: float | httpx.Timeout | None = None, + rebuild_without_native_output: Callable[[], Mapping[str, object]] | None = None, ) -> httpx.Response: max_attempts: Final = max(provider_config.max_retry_on_anthropic_messages_http_error, 1) litellm_params_dict: Final = dict(litellm_params) optional_params_dict: Final = dict(litellm_params) - for attempt_idx in range(max_attempts): + recovered_errors: tuple[httpx.HTTPStatusError, ...] = () # rebind-ok: a rebuilt body replays every recovery + while True: try: response = await async_httpx_client.post( url=request_url, @@ -2116,40 +2122,54 @@ class BaseLLMHTTPHandler: response.raise_for_status() return response except httpx.HTTPStatusError as e: - hit_max_attempt = attempt_idx + 1 == max_attempts + rebuild_body = ( + rebuild_without_native_output + if rebuild_without_native_output is not None + and provider_config.should_retry_anthropic_messages_without_native_output( + e=e, request_body=request_body, model=model + ) + else None + ) + hit_max_attempt = len(recovered_errors) + 1 == max_attempts should_retry = provider_config.should_retry_anthropic_messages_on_http_error( e=e, litellm_params=litellm_params_dict, request_body=request_body ) - if should_retry and not hit_max_attempt: - if logging_obj.baseline_cache_context is not None: - await logging_obj.invalidate_baseline_cache_estimate("retried_request") + if rebuild_body is None and (hit_max_attempt or not should_retry): + raise self._handle_error(e=e, provider_config=provider_config) + if logging_obj.baseline_cache_context is not None: + await logging_obj.invalidate_baseline_cache_estimate("retried_request") + if rebuild_body is not None: + request_body.clear() + request_body.update(rebuild_body()) + for recovered_error in recovered_errors: + provider_config.transform_anthropic_messages_request_on_http_error( + e=recovered_error, request_data=request_body + ) + else: verbose_logger.debug( "Anthropic /v1/messages: invalid thinking signature; " "stripping thinking blocks and retrying (attempt %s/%s).", - attempt_idx + 2, + len(recovered_errors) + 2, max_attempts, ) provider_config.transform_anthropic_messages_request_on_http_error(e=e, request_data=request_body) - headers, signed_json_body = await sign_request_off_loop_if_aws( - provider_config, - provider_config.sign_request, - headers=headers, - optional_params=optional_params_dict, - request_data=request_body, - api_base=request_url, - api_key=api_key, - stream=stream, - fake_stream=False, - model=model, - ) - logging_obj.model_call_details.update(request_body) - continue - raise self._handle_error(e=e, provider_config=provider_config) + recovered_errors = (*recovered_errors, e) + headers, signed_json_body = await sign_request_off_loop_if_aws( + provider_config, + provider_config.sign_request, + headers=headers, + optional_params=optional_params_dict, + request_data=request_body, + api_base=request_url, + api_key=api_key, + stream=stream, + fake_stream=False, + model=model, + ) + logging_obj.model_call_details.update(request_body) except Exception as e: raise self._handle_error(e=e, provider_config=provider_config) - raise RuntimeError("unreachable: anthropic messages HTTP retry loop exited without return") - @staticmethod def _resolve_anthropic_messages_timeout( litellm_params: GenericLiteLLMParams, @@ -2270,6 +2290,24 @@ class BaseLLMHTTPHandler: anthropic_messages_optional_request_params, path ) + native_output_fallback_params: Final = anthropic_messages_provider_config.native_output_fallback_litellm_params( + model=model, + anthropic_messages_optional_request_params=_OPTIONAL_PARAMS.validate_python(anthropic_messages_optional_request_params), + litellm_params=litellm_params, + ) + rebuild_without_native_output: Final = ( + None + if native_output_fallback_params is None + else partial[Mapping[str, object]]( + anthropic_messages_provider_config.transform_anthropic_messages_request, + model=model, + messages=copy.deepcopy(_JSON_OBJECTS.validate_python(messages)), + anthropic_messages_optional_request_params=copy.deepcopy(_JSON_OBJECT.validate_python(anthropic_messages_optional_request_params)), + litellm_params=native_output_fallback_params, + headers=_JSON_OBJECT.validate_python(headers), + ) + ) + # Prepare request body request_body: Final = anthropic_messages_provider_config.transform_anthropic_messages_request( model=model, @@ -2343,6 +2381,7 @@ class BaseLLMHTTPHandler: stream=stream or False, custom_llm_provider=custom_llm_provider, ), + rebuild_without_native_output=rebuild_without_native_output, ) # used for logging + cost tracking diff --git a/litellm/main.py b/litellm/main.py index eee709d95da..76e449aec99 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -121,7 +121,10 @@ from litellm.llms.base_llm.base_model_iterator import ( from litellm.llms.base_llm.chat.transformation import with_attribution_headers from litellm.llms.bedrock.common_utils import ( BedrockModelInfo, + NativeOutputFallback, bedrock_route_for_request, + deployment_model_info, + without_bedrock_native_output_opt_in, without_bedrock_route_prefix, ) from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config @@ -219,6 +222,9 @@ from .llms.azure.completion.handler import AzureTextCompletion from .llms.azure_ai.anthropic.handler import AzureAnthropicChatCompletion from .llms.azure_ai.embed import AzureAIEmbedding from .llms.bedrock.chat import BedrockConverseLLM +from .llms.bedrock.chat.converse_transformation import AmazonConverseConfig +from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import AmazonAnthropicClaudeConfig +from .llms.bedrock.chat.native_output_fallback import NativeOutputAttemptResult, call_with_native_output_fallback from .llms.bedrock.embed.embedding import BedrockEmbedding from .llms.bedrock.image_edit.handler import BedrockImageEdit from .llms.bedrock.image_generation.image_handler import BedrockImageGeneration @@ -4188,6 +4194,43 @@ _OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS: Final = ( + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "aws_region_name", +) + + +def _native_output_fallback( + ctx: CompletionDispatchContext, + first_attempt_params: Mapping[str, object], + first_attempt_litellm_params: Mapping[str, object], +) -> NativeOutputFallback: + litellm_params: Final = without_bedrock_native_output_opt_in(first_attempt_litellm_params) + remapped_params: Final = ctx.optional_params_for_model_info(deployment_model_info(litellm_params)) + return NativeOutputFallback( + optional_params={ + **{key: value for key, value in remapped_params.items() if key != "aws_bedrock_client"}, + **{ + key: first_attempt_params[key] + for key in _AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS + if key in first_attempt_params + }, + }, + litellm_params=litellm_params, + ) + + +def _converse_native_output_fallback( + ctx: CompletionDispatchContext, + first_attempt_params: Mapping[str, object], + first_attempt_litellm_params: Mapping[str, object], +) -> NativeOutputFallback | None: + fallback: Final = _native_output_fallback(ctx, first_attempt_params, first_attempt_litellm_params) + return None if AmazonConverseConfig.sends_native_text_format(fallback.optional_params) else fallback + + def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: acompletion: Final = ctx.acompletion api_base: Final = ctx.api_base @@ -4259,22 +4302,36 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu ) elif bedrock_route == "converse": model = without_bedrock_route_prefix(model) - response = bedrock_converse_chat_completion.completion( - model=model, - messages=messages, - custom_prompt_dict=custom_prompt_dict, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - extra_headers=headers, # Use merged headers instead of original extra_headers - timeout=timeout, - acompletion=acompletion, - client=client, - api_base=api_base, - api_key=api_key, + converse_model: Final = model + + def converse_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult: + return bedrock_converse_chat_completion.completion( + model=converse_model, + messages=messages, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + optional_params=optional_params if fallback is None else dict(fallback.optional_params), + litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params), + logger_fn=logger_fn, + encoding=_get_encoding(), + logging_obj=logging, + extra_headers=headers, # Use merged headers instead of original extra_headers + timeout=timeout, + acompletion=acompletion, + client=client, + api_base=api_base, + api_key=api_key, + ) + + converse_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params) + response = call_with_native_output_fallback( + converse_attempt, + ( + partial(_converse_native_output_fallback, ctx, converse_params, dict(litellm_params)) + if AmazonConverseConfig().uses_opt_in_native_output(converse_model, converse_params, litellm_params) + else None + ), + converse_model, ) elif bedrock_route == "converse_like": model = model.replace("converse_like/", "") @@ -4296,22 +4353,36 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu client=client, ) else: - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - custom_llm_provider="bedrock", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, + + def invoke_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult: + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params if fallback is None else dict(fallback.optional_params), + litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params), + custom_llm_provider="bedrock", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) + + invoke_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params) + response = call_with_native_output_fallback( + invoke_attempt, + ( + partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params)) + if isinstance(provider_config, AmazonAnthropicClaudeConfig) + and provider_config.uses_opt_in_native_output(model, invoke_params, litellm_params) + else None + ), + model, ) return response @@ -5657,10 +5728,14 @@ def completion( "allowed_openai_params": allowed_openai_params, "base_model": base_model, } - optional_params = get_optional_params( - **optional_param_args, - model_info=_MODEL_INFO_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else None, - **non_default_params, + mapping_args: Final = MappingProxyType(dict(optional_param_args)) + mapping_special_params: Final = MappingProxyType(dict(non_default_params)) + + def optional_params_for_model_info(deployment_info: Mapping[str, object] | None) -> dict[str, object]: + return get_optional_params(**mapping_args, model_info=deployment_info, **mapping_special_params) + + optional_params = optional_params_for_model_info( + _MODEL_INFO_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else None ) processed_non_default_params: Final = pre_process_non_default_params( model=model, @@ -5883,6 +5958,7 @@ def completion( model=model, model_response=model_response, optional_params=optional_params, + optional_params_for_model_info=optional_params_for_model_info, organization=organization, provider_config=provider_config, request_params=MappingProxyType( diff --git a/litellm/types/completion.py b/litellm/types/completion.py index 97b77b4ca01..361eeb3426a 100644 --- a/litellm/types/completion.py +++ b/litellm/types/completion.py @@ -229,6 +229,7 @@ class CompletionDispatchContext: model: str model_response: ModelResponse optional_params: dict + optional_params_for_model_info: Callable[[Mapping[str, object] | None], dict[str, object]] organization: str | None provider_config: BaseConfig | None request_params: Mapping[str, object] diff --git a/tests/integration/providers/test_bedrock_batch_files_wire.py b/tests/integration/providers/test_bedrock_batch_files_wire.py index 1a834fc2f8c..823bcda7e06 100644 --- a/tests/integration/providers/test_bedrock_batch_files_wire.py +++ b/tests/integration/providers/test_bedrock_batch_files_wire.py @@ -2,7 +2,7 @@ import json from typing import Final import pytest -from integration._support.client import Gateway +from integration._support.client import JSON_OBJECT, Gateway, list_value, object_value from integration._support.wire import Reply, Request, wire_server MODEL: Final = "bedrock/anthropic.claude-3-haiku-20240307-v1:0" @@ -76,3 +76,56 @@ def test_completions_and_responses_batch_records_upload_as_anthropic_user_messag assert len(uploads) == 1, f"Expected exactly one S3 PUT, saw {[upload.target for upload in uploads]}" stored: Final = tuple(json.loads(line) for line in uploads[0].body.decode().splitlines() if line.strip()) assert stored == EXPECTED_S3_OBJECT + + +_JSON_TOOL: Final = "json_tool_call" +_SCHEMA: Final = { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, +} + + +@pytest.mark.parametrize( + "backend", ["us.anthropic.claude-haiku-5-5", "us.anthropic.claude-opus-4-8"], ids=["haiku_5_5", "opus_4_8"] +) +def test_json_schema_batch_record_with_thinking_uploads_the_json_tool_and_no_native_output( + gateway: Gateway, backend: str +) -> None: + record: Final = { + "custom_id": "schema-record", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": f"bedrock/{backend}", + "messages": [{"role": "user", "content": "Name a city."}], + "max_tokens": 2048, + "thinking": {"type": "adaptive"}, + "response_format": {"type": "json_schema", "json_schema": {"name": "city", "schema": _SCHEMA}}, + }, + } + with wire_server(s3_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"bedrock/{backend}", + api_key=None, + api_base=None, + aws_access_key_id="AKIAIOSFODNN7EXAMPLE", + aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + aws_region_name="us-east-1", + s3_bucket_name=BUCKET, + s3_endpoint_url=wire.url, + ) + response: Final = gateway.request_multipart( + "/v1/files", + {"purpose": "batch", "model": model}, + {"file": ("in.jsonl", (json.dumps(record) + "\n").encode(), "application/jsonl")}, + ) + assert response.status_code == 200, response.text + (upload,) = wire.drain() + (stored,) = tuple(JSON_OBJECT.validate_json(line) for line in upload.body.decode().splitlines() if line.strip()) + model_input: Final = object_value(stored["modelInput"]) + assert "format" not in object_value(model_input.get("output_config", {})), model_input + (tool,) = (object_value(entry) for entry in list_value(model_input["tools"])) + assert tool["name"] == _JSON_TOOL, model_input + assert object_value(tool["input_schema"])["properties"] == _SCHEMA["properties"], model_input diff --git a/tests/integration/providers/test_bedrock_native_structured_output_wire.py b/tests/integration/providers/test_bedrock_native_structured_output_wire.py index f389e53d132..5803d1dee44 100644 --- a/tests/integration/providers/test_bedrock_native_structured_output_wire.py +++ b/tests/integration/providers/test_bedrock_native_structured_output_wire.py @@ -1,14 +1,16 @@ +import base64 import json import re import uuid -from collections.abc import Iterable, Iterator, Mapping +from collections.abc import Callable, Iterable, Iterator, Mapping from itertools import chain from typing import Final from urllib.parse import unquote import httpx import pytest -from integration._support.client import Gateway, Scenario, list_value, object_value, string_value +from integration._support.client import Gateway, Scenario, held, list_value, object_value, string_value +from integration._support.database import read_rows from integration._support.upstream import aws_event_stream_frame from integration._support.wire import Reply, Request, Wire, wire_server from pydantic import JsonValue, TypeAdapter @@ -17,7 +19,8 @@ _HAIKU: Final = "us.anthropic.claude-haiku-5-5" _FABLE: Final = "us.anthropic.claude-fable-5" _SONNET: Final = "us.anthropic.claude-sonnet-4-6" _OPUS: Final = "us.anthropic.claude-opus-5-5" -_OPT_IN: Final = "au.anthropic.claude-opus-4-8" +_OPT_IN: Final = _OPUS +_OPUS_4_8: Final = "us.anthropic.claude-opus-4-8" _DEEPSEEK: Final = "deepseek.v3.2" _AWS: Final[dict[str, JsonValue]] = { "aws_access_key_id": "AKIASCRIPTEDPROVIDER", @@ -38,6 +41,20 @@ _JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) _FABLE_REJECTION: Final = ( "The model returned the following errors: output_config.format: Extra inputs are not permitted" ) +_SCHEMA_TOO_COMPLEX: Final = ( + "The model returned the following errors: compilation exceeded limits: Schema is too complex." +) +_PATTERN_TOO_COMPLEX: Final = ( + "The model returned the following errors: output_config.format.schema: Unsupported regex feature in pattern " + "field: pattern is too complex for structured output: reduce the {n,m} upper bound, narrow the character class " + "range, or avoid nesting quantified groups" +) +_MAX_TOKENS_400: Final = ( + "The model returned the following errors: max_tokens: 2048 > 1024, the maximum for this request" +) +_SIGNATURE_400: Final = ( + "The model returned the following errors: messages.1.content.0: Invalid `signature` in `thinking` block" +) _UNSUPPORTED_KEYWORDS: Final = ( "minimum", "maximum", @@ -327,6 +344,54 @@ def _converse_frames(blocks: tuple[dict[str, JsonValue], ...], stop_reason: str) ) +def _invoke_chunk(event: Mapping[str, JsonValue]) -> bytes: + return aws_event_stream_frame( + {":event-type": "chunk", ":content-type": "application/json", ":message-type": "event"}, + json.dumps({"bytes": base64.b64encode(json.dumps(event).encode()).decode()}).encode(), + ) + + +def _invoke_block_events(index: int, block: Mapping[str, JsonValue]) -> tuple[dict[str, JsonValue], ...]: + def start(content_block: dict[str, JsonValue]) -> dict[str, JsonValue]: + return {"type": "content_block_start", "index": index, "content_block": content_block} + + def delta(payload: dict[str, JsonValue]) -> dict[str, JsonValue]: + return {"type": "content_block_delta", "index": index, "delta": payload} + + stop: Final[dict[str, JsonValue]] = {"type": "content_block_stop", "index": index} + match block.get("type"): + case "thinking": + return ( + start({"type": "thinking", "thinking": ""}), + delta({"type": "thinking_delta", "thinking": block["thinking"]}), + delta({"type": "signature_delta", "signature": block["signature"]}), + stop, + ) + case "tool_use": + return ( + start({"type": "tool_use", "id": block["id"], "name": block["name"], "input": {}}), + delta({"type": "input_json_delta", "partial_json": json.dumps(block["input"])}), + stop, + ) + case _: + return (start({"type": "text", "text": ""}), delta({"type": "text_delta", "text": block["text"]}), stop) + + +def _invoke_frames(message: Mapping[str, JsonValue]) -> bytes: + blocks: Final = tuple(object_value(block) for block in list_value(message["content"])) + events: Final[tuple[Mapping[str, JsonValue], ...]] = ( + {"type": "message_start", "message": {**message, "content": [], "stop_reason": None}}, + *chain.from_iterable(_invoke_block_events(index, block) for index, block in enumerate(blocks)), + { + "type": "message_delta", + "delta": {"stop_reason": message["stop_reason"], "stop_sequence": None}, + "usage": {"output_tokens": 7}, + }, + {"type": "message_stop"}, + ) + return b"".join(_invoke_chunk(event) for event in events) + + def _called_tool(tools: Iterable[str], native: bool) -> str | None: names: Final = frozenset(tools) if _USER_TOOL in names: @@ -365,6 +430,8 @@ def _bedrock(request: Request) -> Reply: "stop_sequence": None, "usage": {"input_tokens": 11, "output_tokens": 7}, } + if _is_stream(request): + return Reply(body=_invoke_frames(message), content_type=_EVENT_STREAM) return Reply(body=json.dumps(message).encode()) if _is_stream(request): return Reply(body=_converse_frames(blocks, stop_reason), content_type=_EVENT_STREAM) @@ -446,6 +513,20 @@ def _messages_text(blocks: tuple[dict[str, JsonValue], ...]) -> JsonValue: return _JSON_VALUE.validate_json(string_value(text["text"])) +def _messages_stream_text(response: httpx.Response) -> JsonValue: + assert response.status_code == 200, response.text + events: Final = tuple( + _JSON.validate_json(line.removeprefix("data: ")) + for line in response.text.splitlines() + if line.startswith("data: {") + ) + deltas: Final = tuple( + object_value(event["delta"]) for event in events if event.get("type") == "content_block_delta" + ) + text: Final = "".join(string_value(delta["text"]) for delta in deltas if delta.get("type") == "text_delta") + return _JSON_VALUE.validate_json(text) + + def _refs(schema: JsonValue) -> frozenset[str]: return frozenset(string_value(node["$ref"]) for node in _schema_nodes(schema) if "$ref" in node) @@ -551,15 +632,18 @@ def test_bedrock_native_model_without_thinking_keeps_the_forced_json_tool(gatewa @pytest.mark.parametrize( - ("backend", "model_info"), - [(_OPUS, None), (_OPT_IN, {"supports_bedrock_native_structured_output": True})], - ids=["neither_key", "deployment_opt_in"], + ("backend", "model_info", "native"), + [ + (_OPUS, None, False), + (_OPT_IN, {"supports_bedrock_native_structured_output": True}, True), + (_OPUS_4_8, {"supports_bedrock_native_structured_output": False}, False), + ], + ids=["neither_key", "deployment_opt_in", "deployment_opt_out"], ) def test_thinking_request_follows_the_bedrock_native_key_the_model_resolves_to( - gateway: Gateway, backend: str, model_info: Mapping[str, JsonValue] | None + gateway: Gateway, backend: str, model_info: Mapping[str, JsonValue] | None, native: bool ) -> None: marker, marker_hex = _marker() - native: Final = model_info is not None with wire_server(_bedrock) as wire, gateway.scenario() as scenario: model: Final = _deployment(scenario, wire, "converse", backend, model_info) response: Final = gateway.request( @@ -818,3 +902,293 @@ def test_numeric_formats_and_wide_schemas_follow_the_bedrock_native_key( tools: Final = _tools(body, invoke) assert (_JSON_TOOL in tools) is not native, body assert native or tools[_JSON_TOOL]["properties"] == schema["properties"], body + + +def _rejecting_native_output(message: str) -> Callable[[Request], Reply]: + def reply(request: Request) -> Reply: + native: Final = _native_schema(_JSON.validate_json(request.body), _is_invoke(request)) + return _bedrock(request) if native is None else _error_reply(message) + + return reply + + +def _fallback_answer(gateway: Gateway, entry: str, model: str, marker: str) -> JsonValue: + match entry: + case "converse_chat_stream" | "invoke_chat_stream": + streamed: Final = _chat_body( + model, marker, stream=True, thinking=_ADAPTIVE, response_format=_response_format(_SCHEMA) + ) + return _content_json(_stream_message(gateway.request("POST", "/v1/chat/completions", streamed))) + case "invoke_messages_stream": + streamed_messages: Final = _messages_body( + model, marker, stream=True, output_config={"format": {"type": "json_schema", "schema": _SCHEMA}} + ) + return _messages_stream_text(gateway.request("POST", "/v1/messages", streamed_messages)) + case "converse_output_config": + chat: Final = _chat_body( + model, marker, output_config={"format": {"type": "json_schema", "schema": _SCHEMA}} + ) + return _content_json(_chat_message(gateway.request("POST", "/v1/chat/completions", chat))) + case _: + return _opt_in_answer(gateway, entry, model, marker) + + +@pytest.mark.parametrize( + ("entry", "rejection"), + [ + ("converse_chat", _FABLE_REJECTION), + ("converse_chat", _PATTERN_TOO_COMPLEX), + ("converse_chat_stream", _SCHEMA_TOO_COMPLEX), + ("converse_responses", _FABLE_REJECTION), + ("converse_messages", _FABLE_REJECTION), + ("converse_output_config", _FABLE_REJECTION), + ("invoke_chat", _FABLE_REJECTION), + ("invoke_chat_stream", _FABLE_REJECTION), + ("invoke_messages", _SCHEMA_TOO_COMPLEX), + ("invoke_messages_stream", _FABLE_REJECTION), + ], +) +def test_bedrock_rejecting_opt_in_native_output_gets_one_retry_without_it( + gateway: Gateway, entry: str, rejection: str +) -> None: + marker, marker_hex = _marker() + route: Final = entry.split("_", 1)[0] + invoke: Final = route == "invoke" + with wire_server(_rejecting_native_output(rejection)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, route, _OPUS_4_8) + assert _fallback_answer(gateway, entry, model, marker) == _payload(marker_hex) + rejected, retried = (_JSON.validate_json(request.body) for request in wire.drain()) + assert _native_schema(rejected, invoke) is not None, rejected + assert _native_schema(retried, invoke) is None, retried + tools: Final = _tools(retried, invoke) + match entry: + case "converse_output_config": + assert not tools, retried + case "invoke_messages" | "invoke_messages_stream": + assert not tools, retried + prompt: Final = json.dumps(retried["messages"]) + assert all(name in prompt for name in object_value(_SCHEMA["properties"])), retried + case _: + assert tools[_JSON_TOOL]["properties"] == _SCHEMA["properties"], retried + + +@pytest.mark.parametrize( + ("backend", "rejection"), + [(_OPUS_4_8, _MAX_TOKENS_400), (_SONNET, _FABLE_REJECTION)], + ids=["other_400", "native_model"], +) +@pytest.mark.parametrize("route", ["converse", "invoke"]) +def test_bedrock_400_is_not_retried_unless_it_rejects_opt_in_native_output( + gateway: Gateway, route: str, backend: str, rejection: str +) -> None: + marker, marker_hex = _marker() + with wire_server(_rejecting_native_output(rejection)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, route, backend) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + _chat_body(model, marker, thinking=_ADAPTIVE, response_format=_response_format(_SCHEMA)), + ) + assert response.status_code == 400, response.text + assert rejection.split(": ", 1)[1] in response.text, response.text + _, body = _one_call(wire, marker_hex) + assert _native_schema(body, route == "invoke") == _SCHEMA, body + + +def _always_rejecting(message: str) -> Callable[[Request], Reply]: + def reply(_: Request) -> Reply: + return _error_reply(message) + + return reply + + +@pytest.mark.parametrize( + ("backend", "rejection"), + [(_OPUS_4_8, _MAX_TOKENS_400), (_SONNET, _FABLE_REJECTION)], + ids=["other_400", "native_model"], +) +def test_messages_400_on_bedrock_invoke_reaches_the_caller_after_one_call_unless_it_rejects_opt_in_native_output( + gateway: Gateway, backend: str, rejection: str +) -> None: + marker, marker_hex = _marker() + with wire_server(_always_rejecting(rejection)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, "invoke", backend) + response: Final = gateway.request( + "POST", + "/v1/messages", + _messages_body(model, marker, output_config={"format": {"type": "json_schema", "schema": _SCHEMA}}), + ) + assert response.status_code == 400, response.text + assert rejection.split(": ", 1)[1] in response.text, response.text + _one_call(wire, marker_hex) + + +def test_a_messages_retry_without_native_output_still_omits_the_deployments_dropped_params(gateway: Gateway) -> None: + marker, marker_hex = _marker() + with wire_server(_rejecting_native_output(_FABLE_REJECTION)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"bedrock/invoke/{_OPUS_4_8}", + api_base=wire.url, + model_info=None, + additional_drop_params=["stop_sequences"], + **_AWS, + ) + request: Final = _messages_body( + model, marker, stop_sequences=["END"], output_config={"format": {"type": "json_schema", "schema": _SCHEMA}} + ) + assert _messages_text(_messages_blocks(gateway.request("POST", "/v1/messages", request))) == _payload( + marker_hex + ) + rejected, retried = (_JSON.validate_json(call.body) for call in wire.drain()) + assert _native_schema(rejected, True) is not None, rejected + assert _native_schema(retried, True) is None, retried + assert "stop_sequences" not in rejected, rejected + assert "stop_sequences" not in retried, retried + + +def _carries_thinking_block(message: JsonValue) -> bool: + content: Final = object_value(message).get("content") + return isinstance(content, list) and any(object_value(block).get("type") == "thinking" for block in content) + + +def _replays_thinking(body: Mapping[str, JsonValue]) -> bool: + return any(_carries_thinking_block(message) for message in list_value(body["messages"])) + + +def _rejecting_native_output_and_foreign_signatures(signature_first: bool) -> Callable[[Request], Reply]: + def reply(request: Request) -> Reply: + body: Final = _JSON.validate_json(request.body) + checks: Final = ( + (_replays_thinking(body), _SIGNATURE_400), + (_native_schema(body, True) is not None, _FABLE_REJECTION), + ) + failing: Final = tuple(message for failed, message in (checks if signature_first else checks[::-1]) if failed) + return _error_reply(failing[0]) if failing else _bedrock(request) + + return reply + + +@pytest.mark.parametrize("signature_first", [False, True], ids=["rejection_first", "signature_first"]) +def test_messages_retry_without_native_output_keeps_the_thinking_signature_fix( + gateway: Gateway, signature_first: bool +) -> None: + marker, marker_hex = _marker() + history: Final[list[JsonValue]] = [ + {"role": "user", "content": [{"type": "text", "text": f"first question {marker}"}]}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "from another deployment", "signature": _SIGNATURE}, + {"type": "text", "text": "first answer"}, + ], + }, + {"role": "user", "content": [{"type": "text", "text": "second question"}]}, + ] + with ( + wire_server(_rejecting_native_output_and_foreign_signatures(signature_first)) as wire, + gateway.scenario() as scenario, + ): + model: Final = _deployment(scenario, wire, "invoke", _OPUS_4_8) + request: Final = { + **_messages_body( + model, marker, thinking=_ADAPTIVE, output_config={"format": {"type": "json_schema", "schema": _SCHEMA}} + ), + "messages": history, + } + assert _messages_text(_messages_blocks(gateway.request("POST", "/v1/messages", request))) == _payload( + marker_hex + ) + sent: Final = tuple(_JSON.validate_json(call.body) for call in wire.drain()) + assert len(sent) == 3, sent + assert _native_schema(sent[0], True) is not None and _replays_thinking(sent[0]), sent[0] + assert _native_schema(sent[-1], True) is None and not _replays_thinking(sent[-1]), sent[-1] + assert all(name in json.dumps(sent[-1]["messages"]) for name in object_value(_SCHEMA["properties"])), sent[-1] + + +@pytest.mark.parametrize("entry", ["converse_chat", "invoke_messages"]) +def test_a_request_retried_without_native_output_lands_one_success_spend_row(gateway: Gateway, entry: str) -> None: + marker, marker_hex = _marker() + invoke: Final = entry.startswith("invoke") + with wire_server(_rejecting_native_output(_FABLE_REJECTION)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, entry.split("_", 1)[0], _OPUS_4_8) + assert _fallback_answer(gateway, entry, model, marker) == _payload(marker_hex) + sent: Final = tuple(_JSON.validate_json(call.body) for call in wire.drain()) + assert len(sent) == (1 if _native_schema(sent[0], invoke) is None else 2), sent + rows: Final = held( + lambda: read_rows('SELECT status FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)), + lambda found: len(found) == 1, + holding=2, + seconds=70, + ) + assert rows[0]["status"] == "success", rows + + +_CALLER_OUTPUT_CONFIG_WITH_A_SCHEMA_OBJECT: Final[dict[str, JsonValue]] = { + "textFormat": {"type": "json_schema", "structure": {"jsonSchema": {"name": "risk", "schema": {"type": "object"}}}} +} +_BEDROCK_SCHEMA_OBJECT_400: Final = "Start of structure or map found where not expected" + + +def _converse_json_schema_field(body: Mapping[str, JsonValue]) -> JsonValue: + text_format: Final = object_value(body.get("outputConfig") or {}).get("textFormat") or {} + structure: Final = object_value(text_format).get("structure") or {} + return object_value(object_value(structure).get("jsonSchema") or {}).get("schema") + + +def _deserializing_like_bedrock(request: Request) -> Reply: + schema: Final = _converse_json_schema_field(_JSON.validate_json(request.body)) + if schema is None or isinstance(schema, str): + return _bedrock(request) + return Reply(status=400, body=json.dumps({"Message": _BEDROCK_SCHEMA_OBJECT_400}).encode()) + + +def test_a_malformed_caller_output_config_on_an_opt_in_model_gets_bedrocks_own_400_after_one_call( + gateway: Gateway, +) -> None: + marker, marker_hex = _marker() + with wire_server(_deserializing_like_bedrock) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, "converse", _OPUS_4_8) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + _chat_body( + model, + marker, + thinking=_ADAPTIVE, + response_format=_response_format(_SCHEMA), + outputConfig=_CALLER_OUTPUT_CONFIG_WITH_A_SCHEMA_OBJECT, + ), + ) + assert response.status_code == 400, response.text + assert _BEDROCK_SCHEMA_OBJECT_400 in response.text, response.text + _, body = _one_call(wire, marker_hex) + assert body["outputConfig"] == _CALLER_OUTPUT_CONFIG_WITH_A_SCHEMA_OBJECT, body + + +_CALLER_OUTPUT_CONFIG: Final[dict[str, JsonValue]] = { + "textFormat": {"type": "json_schema", "structure": {"jsonSchema": {"name": "risk", "schema": json.dumps(_SCHEMA)}}} +} + + +@pytest.mark.parametrize("thinking", [None, _ADAPTIVE], ids=["no_thinking", "adaptive_thinking"]) +def test_a_caller_output_config_bedrock_refuses_is_sent_once_on_an_opt_in_model( + gateway: Gateway, thinking: dict[str, JsonValue] | None +) -> None: + marker, marker_hex = _marker() + with wire_server(_rejecting_native_output(_PATTERN_TOO_COMPLEX)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, "converse", _OPUS_4_8) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + _chat_body( + model, + marker, + response_format=_response_format(_SCHEMA), + outputConfig=_CALLER_OUTPUT_CONFIG, + **({} if thinking is None else {"thinking": thinking}), + ), + ) + assert response.status_code == 400, response.text + assert "pattern is too complex for structured output" in response.text, response.text + _, body = _one_call(wire, marker_hex) + assert body["outputConfig"] == _CALLER_OUTPUT_CONFIG, body diff --git a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py index b2aba7fb100..bba79049283 100644 --- a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py +++ b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py @@ -757,12 +757,15 @@ def _vertex_deployment(gateway: Gateway, scenario: Scenario, api_base: str, back ) -def _bedrock_invoke_deployment(scenario: Scenario, api_base: str, backend: str) -> str: +def _bedrock_invoke_deployment( + scenario: Scenario, api_base: str, backend: str, model_info: Mapping[str, JsonValue] | None = None +) -> str: return scenario.model( model=f"bedrock/invoke/{backend}", api_base=api_base, api_key=_BEDROCK_TOKEN, aws_region_name="us-east-1", + model_info=model_info, ) @@ -1521,7 +1524,9 @@ def _json_tool_deployment(gateway: Gateway, scenario: Scenario, api_base: str, p case "vertex_ai": return _vertex_deployment(gateway, scenario, api_base, _TOOL_MODEL) case "bedrock_invoke": - return _bedrock_invoke_deployment(scenario, api_base, _BEDROCK_JSON_TOOL_MODEL) + return _bedrock_invoke_deployment( + scenario, api_base, _BEDROCK_JSON_TOOL_MODEL, {"supports_bedrock_native_structured_output": False} + ) case _: raise AssertionError(provider) diff --git a/tests/integration/sdk/test_bedrock_converse_stream_sync_decoder_wire.py b/tests/integration/sdk/test_bedrock_converse_stream_sync_decoder_wire.py index 90acdd774d0..87efbc5529e 100644 --- a/tests/integration/sdk/test_bedrock_converse_stream_sync_decoder_wire.py +++ b/tests/integration/sdk/test_bedrock_converse_stream_sync_decoder_wire.py @@ -98,3 +98,17 @@ def test_k03_sync_stream_with_normal_frames_delivers_the_text_and_a_stop() -> No assert consumed.text == _ANSWER, consumed assert consumed.finish_reasons[-1] == "stop", consumed assert len(wire.drain()) == 1 + + +def test_k04_sync_stream_open_400_carries_the_bedrock_body_as_text() -> None: + body: Final = '{"message":"The request is invalid."}' + + def reject(_: Request) -> Reply: + return Reply(status=400, body=body.encode()) + + with wire_server(reject) as wire: + with pytest.raises(BadRequestError) as raised: + _consume_sync_stream(wire) + assert raised.value.status_code == 400, raised.value + assert f"BedrockException - {body}" in raised.value.message, raised.value.message + assert len(wire.drain()) == 1 diff --git a/tests/integration/sdk/test_bedrock_native_output_fallback_sdk.py b/tests/integration/sdk/test_bedrock_native_output_fallback_sdk.py new file mode 100644 index 00000000000..745c1ceb603 --- /dev/null +++ b/tests/integration/sdk/test_bedrock_native_output_fallback_sdk.py @@ -0,0 +1,336 @@ +import json +import re +import uuid +from collections.abc import Mapping +from typing import Final + +import boto3 +import pytest +from integration._support.client import list_value, object_value, string_value +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +import litellm +from litellm import CustomStreamWrapper, ModelResponse +from litellm.types.utils import Choices, StreamingChoices + +_OPUS_4_8: Final = "us.anthropic.claude-opus-4-8" +_JSON_TOOL: Final = "json_tool_call" +_MARKER: Final = re.compile(r"probe ([0-9a-f]{32})") +_REJECTION: Final = "The model returned the following errors: output_config.format: Extra inputs are not permitted" +_SCHEMA: Final = { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, +} +_JSON: Final = TypeAdapter(dict[str, JsonValue]) +_ROUTES: Final = pytest.mark.parametrize( + ("route", "stream"), + [("converse", False), ("converse", True), ("invoke", False)], + ids=["converse", "converse_stream", "invoke"], +) + + +@pytest.fixture +def marker_hex() -> str: + return uuid.uuid4().hex + + +def _is_invoke(request: Request) -> bool: + return "/invoke" in request.target + + +def _native(body: Mapping[str, JsonValue], invoke: bool) -> bool: + output_config: Final = body.get("output_config" if invoke else "outputConfig") + return isinstance(output_config, dict) and ("format" if invoke else "textFormat") in output_config + + +def _tool_names(body: Mapping[str, JsonValue], invoke: bool) -> frozenset[str]: + if invoke: + return frozenset(string_value(object_value(tool)["name"]) for tool in list_value(body.get("tools", []))) + tools: Final = list_value(object_value(body.get("toolConfig") or {"tools": []})["tools"]) + return frozenset(string_value(object_value(object_value(tool)["toolSpec"])["name"]) for tool in tools) + + +def _tool_reply(invoke: bool, answer: Mapping[str, JsonValue]) -> Reply: + if invoke: + message: Final = { + "id": "msg_scripted", + "type": "message", + "role": "assistant", + "model": "claude-scripted", + "content": [{"type": "tool_use", "id": "toolu_scripted", "name": _JSON_TOOL, "input": answer}], + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 11, "output_tokens": 7}, + } + return Reply(body=json.dumps(message).encode()) + converse: Final = { + "output": { + "message": { + "role": "assistant", + "content": [{"toolUse": {"toolUseId": "tooluse_scripted", "name": _JSON_TOOL, "input": answer}}], + } + }, + "stopReason": "tool_use", + "usage": {"inputTokens": 11, "outputTokens": 7, "totalTokens": 18}, + "metrics": {"latencyMs": 1}, + } + return Reply(body=json.dumps(converse).encode()) + + +def _bedrock_without_native_output(request: Request) -> Reply: + body: Final = _JSON.validate_json(request.body) + invoke: Final = _is_invoke(request) + if _native(body, invoke): + return Reply(status=400, body=json.dumps({"message": _REJECTION}).encode()) + found: Final = _MARKER.search(json.dumps(body)) + assert found is not None and _JSON_TOOL in _tool_names(body, invoke), body + return _tool_reply(invoke, {"city": found.group(1)}) + + +def _request(route: str, stream: bool, wire: Wire, marker_hex: str) -> Mapping[str, object]: + return { + "model": f"bedrock/{route}/{_OPUS_4_8}", + "messages": [{"role": "user", "content": f"Name a city. probe {marker_hex}"}], + "api_base": wire.url, + "api_key": "synthetic-bedrock-bearer", + "aws_region_name": "us-east-1", + "max_tokens": 2048, + "thinking": {"type": "adaptive"}, + "response_format": {"type": "json_schema", "json_schema": {"name": "city", "schema": _SCHEMA}}, + "num_retries": 0, + "stream": stream, + } + + +def _message_content(response: ModelResponse) -> str: + (choice,) = response.choices + assert isinstance(choice, Choices), response + return string_value(choice.message.content) + + +def _delta_content(chunk: ModelResponse) -> str: + (choice,) = chunk.choices + assert isinstance(choice, StreamingChoices), chunk + return choice.delta.content or "" + + +def _assert_one_retry_without_native_output(wire: Wire, route: str) -> None: + rejected, retried = (_JSON.validate_json(request.body) for request in wire.drain()) + invoke: Final = route == "invoke" + assert _native(rejected, invoke), rejected + assert not _native(retried, invoke), retried + assert _JSON_TOOL in _tool_names(retried, invoke), retried + + +@_ROUTES +def test_sync_call_retries_a_native_output_rejection_once_with_the_json_tool( + route: str, stream: bool, marker_hex: str +) -> None: + with wire_server(_bedrock_without_native_output) as wire: + response: Final = litellm.completion(**_request(route, stream, wire, marker_hex)) + content: Final = ( + "".join(_delta_content(chunk) for chunk in response) + if isinstance(response, CustomStreamWrapper) + else _message_content(response) + ) + assert json.loads(content) == {"city": marker_hex} + _assert_one_retry_without_native_output(wire, route) + + +@_ROUTES +async def test_async_call_retries_a_native_output_rejection_once_with_the_json_tool( + route: str, stream: bool, marker_hex: str +) -> None: + with wire_server(_bedrock_without_native_output) as wire: + response: Final = await litellm.acompletion(**_request(route, stream, wire, marker_hex)) + content: Final = ( + "".join([_delta_content(chunk) async for chunk in response]) + if isinstance(response, CustomStreamWrapper) + else _message_content(response) + ) + assert json.loads(content) == {"city": marker_hex} + _assert_one_retry_without_native_output(wire, route) + + +_CONSTRAINED_SCHEMA: Final = { + "type": "object", + "properties": { + "city": {"type": "string", "maxLength": 20}, + "population": {"type": "integer", "minimum": 0}, + "districts": {"type": "array", "items": {"type": "string"}, "minItems": 2, "uniqueItems": True}, + "code": {"oneOf": [{"type": "string"}, {"type": "integer"}]}, + }, + "required": ["city", "population", "districts", "code"], + "additionalProperties": False, +} + + +def _response_format(schema: Mapping[str, object]) -> Mapping[str, object]: + return {"type": "json_schema", "json_schema": {"name": "city", "schema": schema}} + + +def _consume(response: ModelResponse | CustomStreamWrapper) -> None: + if isinstance(response, CustomStreamWrapper): + tuple(response) + + +def _sent_bodies( + route: str, + stream: bool, + marker_hex: str, + request_params: Mapping[str, object], + model_info: Mapping[str, object] | None, +) -> tuple[Mapping[str, JsonValue], ...]: + with wire_server(_bedrock_without_native_output) as wire: + request: Final = { + **_request(route, stream, wire, marker_hex), + **{"thinking": None, "response_format": None}, + **request_params, + **({} if model_info is None else {"model_info": model_info}), + } + _consume(litellm.completion(**{key: value for key, value in request.items() if value is not None})) + return tuple(_JSON.validate_json(sent.body) for sent in wire.drain()) + + +@_ROUTES +@pytest.mark.parametrize( + "request_params", + [ + {"reasoning_effort": "low", "response_format": _response_format(_SCHEMA)}, + {"thinking": {"type": "adaptive"}, "response_format": _response_format(_CONSTRAINED_SCHEMA)}, + { + "thinking": {"type": "adaptive"}, + "response_format": _response_format(_SCHEMA), + "output_config": {"format": {"type": "json_schema", "schema": _SCHEMA}}, + }, + { + "thinking": {"type": "adaptive"}, + "output_config": {"effort": "low"}, + "response_format": _response_format(_SCHEMA), + }, + ], + ids=[ + "reasoning_effort", + "constrained_schema", + "response_format_and_output_config_format", + "adaptive_thinking_with_effort", + ], +) +def test_the_retry_sends_what_a_deployment_without_native_output_sends( + route: str, stream: bool, marker_hex: str, request_params: Mapping[str, object] +) -> None: + rejected, retried = _sent_bodies(route, stream, marker_hex, request_params, None) + (opted_out,) = _sent_bodies( + route, stream, marker_hex, request_params, {"supports_bedrock_native_structured_output": False} + ) + + assert _native(rejected, route == "invoke"), rejected + assert retried == opted_out + + +@_ROUTES +def test_the_retry_keeps_the_callers_continuation_message(route: str, stream: bool, marker_hex: str) -> None: + request_params: Final = { + "thinking": {"type": "adaptive"}, + "response_format": _response_format(_SCHEMA), + "messages": [ + {"role": "user", "content": f"Name a city. probe {marker_hex}"}, + {"role": "assistant", "content": "Which country should it be in?"}, + ], + "user_continue_message": {"role": "user", "content": "Any country in South America"}, + } + + rejected, retried = _sent_bodies(route, stream, marker_hex, request_params, None) + (opted_out,) = _sent_bodies( + route, stream, marker_hex, request_params, {"supports_bedrock_native_structured_output": False} + ) + + assert _native(rejected, route == "invoke"), rejected + assert retried == opted_out + + +_MAX_LENGTH_ONE_OF_SCHEMA: Final = { + "type": "object", + "properties": { + "city": {"type": "string", "maxLength": 20}, + "code": {"oneOf": [{"type": "string"}, {"type": "integer"}]}, + }, + "required": ["city", "code"], + "additionalProperties": False, +} + + +def _messages_bedrock_without_native_output(request: Request) -> Reply: + if not _is_invoke(request) or _native(_JSON.validate_json(request.body), invoke=True): + return _bedrock_without_native_output(request) + message: Final = { + "id": "msg_scripted", + "type": "message", + "role": "assistant", + "model": "claude-scripted", + "content": [{"type": "text", "text": '{"city": "Paris"}'}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 11, "output_tokens": 7}, + } + return Reply(body=json.dumps(message).encode()) + + +async def _messages_bodies( + route: str, marker_hex: str, model_info: Mapping[str, object] | None +) -> tuple[Mapping[str, JsonValue], ...]: + with wire_server(_messages_bedrock_without_native_output) as wire: + await litellm.anthropic_messages( + model=f"bedrock/{route}/{_OPUS_4_8}", + messages=[{"role": "user", "content": f"Name a city. probe {marker_hex}"}], + api_base=wire.url, + api_key="synthetic-bedrock-bearer", + aws_region_name="us-east-1", + max_tokens=2048, + thinking={"type": "adaptive"}, + output_config={"effort": "low", "format": {"type": "json_schema", "schema": _MAX_LENGTH_ONE_OF_SCHEMA}}, + **({} if model_info is None else {"model_info": model_info}), + ) + return tuple(_JSON.validate_json(sent.body) for sent in wire.drain()) + + +@pytest.mark.parametrize("route", ["converse", "invoke"]) +async def test_a_messages_retry_sends_what_a_deployment_without_native_output_sends( + route: str, marker_hex: str +) -> None: + rejected, retried = await _messages_bodies(route, marker_hex, None) + (opted_out,) = await _messages_bodies(route, marker_hex, {"supports_bedrock_native_structured_output": False}) + + assert _native(rejected, invoke=route == "invoke"), rejected + assert retried == opted_out + + +@_ROUTES +def test_the_retry_signs_with_the_credentials_of_a_deprecated_aws_bedrock_client( + route: str, stream: bool, marker_hex: str +) -> None: + aws_bedrock_client: Final = boto3.client( + "bedrock-runtime", + region_name="us-west-2", + aws_access_key_id="AKIASYNTHETICCLIENT", + aws_secret_access_key="synthetic-client-secret", + aws_session_token="synthetic-client-session", + ) + with wire_server(_bedrock_without_native_output) as wire: + request: Final = { + key: value + for key, value in _request(route, stream, wire, marker_hex).items() + if key not in {"api_key", "aws_region_name"} + } + _consume(litellm.completion(**request, aws_bedrock_client=aws_bedrock_client)) + sent: Final = wire.drain() + + assert len(sent) == 2 + for attempt in sent: + headers = {name.lower(): value for name, value in attempt.headers.items()} + assert "Credential=AKIASYNTHETICCLIENT/" in headers["authorization"] + assert "/us-west-2/bedrock/aws4_request" in headers["authorization"] + assert headers["x-amz-security-token"] == "synthetic-client-session" diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index ec711019cd2..0a57f6bb227 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -1930,3 +1930,66 @@ class TestBedrockAnthropicCombinedRegressions: user_msg = result["messages"][0] has_cache_control = any(isinstance(c, dict) and "cache_control" in c for c in user_msg["content"]) assert has_cache_control + + +_CITY_SCHEMA: Final = { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, +} + + +_CITY_RESPONSE_FORMAT: Final = {"type": "json_schema", "json_schema": {"name": "city", "schema": _CITY_SCHEMA}} + + +def _invoke_chat_params( + model: str, model_info: Mapping[str, object] | None, non_default_params: Mapping[str, object] +) -> Mapping[str, object]: + return AmazonAnthropicClaudeConfig(model_info=model_info).map_openai_params( + non_default_params=dict(non_default_params), optional_params={}, model=model, drop_params=False + ) + + +@pytest.mark.parametrize( + "thinking_params", + [{"thinking": {"type": "adaptive"}}, {"reasoning_effort": "low"}], + ids=["adaptive", "reasoning_effort"], +) +@pytest.mark.parametrize("model", ["us.anthropic.claude-opus-4-8", "global.anthropic.claude-sonnet-5"]) +def test_invoke_uses_opt_in_native_output_when_the_opt_in_drove_native_output( + local_model_cost_map, model, thinking_params +): + optional_params: Final = _invoke_chat_params( + model, None, {"response_format": _CITY_RESPONSE_FORMAT, **thinking_params} + ) + + assert AmazonAnthropicClaudeConfig().uses_opt_in_native_output(model, optional_params, {}) + + +@pytest.mark.parametrize( + ("model", "model_info", "request_params"), + [ + ("us.anthropic.claude-opus-4-8", None, {"response_format": _CITY_RESPONSE_FORMAT}), + ( + "us.anthropic.claude-opus-4-8", + {"supports_bedrock_native_structured_output": False}, + {"response_format": _CITY_RESPONSE_FORMAT, "thinking": {"type": "adaptive"}}, + ), + ( + "us.anthropic.claude-sonnet-4-6", + None, + {"response_format": _CITY_RESPONSE_FORMAT, "thinking": {"type": "adaptive"}}, + ), + ], + ids=["no_thinking", "deployment_opt_out", "native_model"], +) +def test_invoke_uses_opt_in_native_output_is_false_when_native_output_did_not_come_from_the_opt_in( + local_model_cost_map, model, model_info, request_params +): + optional_params: Final = _invoke_chat_params(model, model_info, request_params) + + assert not AmazonAnthropicClaudeConfig().uses_opt_in_native_output( + model, optional_params, {"model_info": model_info} + ) + diff --git a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py index c025a368e51..40b75b42893 100644 --- a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py +++ b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py @@ -302,3 +302,49 @@ async def test_async_converse_streaming_fails_at_the_request_timeout_not_the_ups def test_sync_converse_streaming_fails_at_the_request_timeout_not_the_upstreams_pace() -> None: with pytest.raises(litellm.Timeout): litellm.completion(client=slow_upstream_sync_client(), **_converse_streaming_kwargs()) + + +_UPSTREAM_ERROR_BODY: Final = '{"message":"The request is invalid."}' + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + "model", ["us.anthropic.claude-sonnet-4-5-20250929-v1:0", "us.anthropic.claude-opus-4-8"], ids=["sonnet", "opus"] +) +@pytest.mark.parametrize( + ("upstream_status", "error_class", "status_code"), + [ + (400, litellm.BadRequestError, 400), + (429, litellm.RateLimitError, 429), + (500, litellm.ServiceUnavailableError, 503), + (503, litellm.ServiceUnavailableError, 503), + ], +) +def test_sync_converse_streaming_upstream_error_carries_the_upstream_body_as_text( + model: str, + upstream_status: int, + error_class: type[litellm.BadRequestError | litellm.RateLimitError | litellm.ServiceUnavailableError], + status_code: int, +) -> None: + client: Final = HTTPHandler( + client=httpx.Client( + transport=httpx.MockTransport( + lambda request: httpx.Response(upstream_status, content=_UPSTREAM_ERROR_BODY.encode()) + ) + ) + ) + + with pytest.raises(error_class) as raised: + litellm.completion( + model=f"bedrock/converse/{model}", + messages=[{"role": "user", "content": "hi"}], + stream=True, + num_retries=0, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + client=client, + ) + + assert raised.value.status_code == status_code + assert raised.value.message == f"litellm.{error_class.__name__}: BedrockException - {_UPSTREAM_ERROR_BODY}" diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 63ce2b87f6b..70844766d9f 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -9904,3 +9904,156 @@ def test_bedrock_nova_web_search_options_ignored_for_non_nova(): system_tool2 = result2.get("systemTool") assert system_tool2 is not None assert system_tool2["name"] == "nova_grounding" + + +def _converse_params( + model: str, model_info: Mapping[str, object] | None, non_default_params: Mapping[str, object] +) -> Mapping[str, object]: + return AmazonConverseConfig().map_openai_params( + non_default_params=dict(non_default_params), + optional_params={}, + model=model, + drop_params=False, + model_info=model_info, + ) + + +_OUTPUT_CONFIG_FORMAT: Final = { + "output_config": {"format": {"type": "json_schema", "schema": CAPITAL_RESPONSE_FORMAT["json_schema"]["schema"]}} +} + + +@pytest.mark.parametrize( + "request_params", + [ + {"response_format": CAPITAL_RESPONSE_FORMAT, "thinking": ADAPTIVE_THINKING}, + {"response_format": CAPITAL_RESPONSE_FORMAT, "reasoning_effort": "low"}, + {"response_format": CAPITAL_RESPONSE_FORMAT, "thinking": {"type": "enabled", "budget_tokens": 2048}}, + _OUTPUT_CONFIG_FORMAT, + ], + ids=["adaptive", "reasoning_effort", "budget", "output_config_format"], +) +@pytest.mark.parametrize("model", ["us.anthropic.claude-opus-4-8", "global.anthropic.claude-sonnet-5"]) +def test_uses_opt_in_native_output_when_the_opt_in_drove_native_output(local_model_cost_map, model, request_params): + optional_params: Final = _converse_params(model, None, request_params) + + assert AmazonConverseConfig().uses_opt_in_native_output(model, optional_params, {"model_info": {"id": "d1"}}) + + +@pytest.mark.parametrize( + ("model", "model_info", "request_params"), + [ + ("us.anthropic.claude-opus-4-8", None, {"response_format": CAPITAL_RESPONSE_FORMAT}), + ( + "us.anthropic.claude-opus-4-8", + {"supports_bedrock_native_structured_output": False}, + {"response_format": CAPITAL_RESPONSE_FORMAT, "thinking": ADAPTIVE_THINKING}, + ), + ( + "us.anthropic.claude-sonnet-4-6", + {"supports_bedrock_native_structured_output": True}, + {"response_format": CAPITAL_RESPONSE_FORMAT, "thinking": ADAPTIVE_THINKING}, + ), + ], + ids=["no_thinking", "deployment_opt_out", "native_model"], +) +def test_uses_opt_in_native_output_is_false_when_native_output_did_not_come_from_the_opt_in( + local_model_cost_map, model, model_info, request_params +): + optional_params: Final = _converse_params(model, model_info, request_params) + + assert not AmazonConverseConfig().uses_opt_in_native_output(model, optional_params, {"model_info": model_info}) + + +def test_uses_opt_in_native_output_reads_the_base_model_of_an_application_inference_profile(local_model_cost_map): + profile: Final = "arn:aws:bedrock:us-east-1:000000000000:application-inference-profile/profile-id" + litellm_params: Final = {"base_model": "bedrock/us.anthropic.claude-opus-4-8"} + optional_params: Final = _converse_params( + "us.anthropic.claude-opus-4-8", + None, + {"response_format": CAPITAL_RESPONSE_FORMAT, "thinking": ADAPTIVE_THINKING}, + ) + + assert AmazonConverseConfig().uses_opt_in_native_output(profile, optional_params, litellm_params) + + +_CALLER_OUTPUT_CONFIG_WITH_A_SCHEMA_OBJECT: Final = { + "textFormat": {"type": "json_schema", "structure": {"jsonSchema": {"name": "capital", "schema": {"type": "object"}}}} +} + + +_BEDROCK_SCHEMA_TYPE_ERROR: Final = "outputConfig.textFormat.structure.jsonSchema.schema must be a string" + + +@pytest.mark.parametrize("model", [_OPUS, _LLAMA], ids=["opt_in", "not_opt_in"]) +@pytest.mark.respx(assert_all_called=True) +def test_a_malformed_caller_output_config_gets_bedrocks_own_400( + model: str, respx_mock: respx.Router, local_model_cost_map: None +) -> None: + route: Final = respx_mock.post(url__regex=r"/converse$").mock( + return_value=httpx.Response(400, json={"message": _BEDROCK_SCHEMA_TYPE_ERROR}) + ) + + with pytest.raises(litellm.BadRequestError, match=re.escape(_BEDROCK_SCHEMA_TYPE_ERROR)): + completion( + model=f"bedrock/converse/{model}", + messages=[{"role": "user", "content": "What is the capital of France?"}], + response_format=CAPITAL_RESPONSE_FORMAT, + outputConfig=_CALLER_OUTPUT_CONFIG_WITH_A_SCHEMA_OBJECT, + num_retries=0, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + client=HTTPHandler(), + ) + + assert route.call_count == 1 + + +_CALLER_OUTPUT_CONFIG: Final = { + "textFormat": { + "type": "json_schema", + "structure": { + "jsonSchema": {"name": "capital", "schema": json.dumps(CAPITAL_RESPONSE_FORMAT["json_schema"]["schema"])} + }, + } +} + + +_BEDROCK_SCHEMA_REFUSAL: Final = "output_config.format.schema: pattern is too complex for structured output" + + +@pytest.mark.parametrize( + ("caller_params", "expected_calls"), + [ + ({"thinking": ADAPTIVE_THINKING}, 2), + ({"outputConfig": _CALLER_OUTPUT_CONFIG}, 1), + ({"outputConfig": _CALLER_OUTPUT_CONFIG, "thinking": ADAPTIVE_THINKING}, 1), + ], + ids=["litellm_native_output", "caller_output_config", "caller_output_config_thinking"], +) +@pytest.mark.respx(assert_all_called=True) +def test_a_refused_native_schema_is_sent_again_only_when_the_retry_drops_native_output( + caller_params: dict, expected_calls: int, respx_mock: respx.Router, local_model_cost_map: None +) -> None: + route: Final = respx_mock.post(url__regex=r"/converse$").mock( + return_value=httpx.Response(400, json={"message": _BEDROCK_SCHEMA_REFUSAL}) + ) + + with pytest.raises(litellm.BadRequestError, match=re.escape(_BEDROCK_SCHEMA_REFUSAL)): + completion( + model=f"bedrock/converse/{_OPUS}", + messages=[{"role": "user", "content": "What is the capital of France?"}], + response_format=CAPITAL_RESPONSE_FORMAT, + num_retries=0, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + client=HTTPHandler(), + **caller_params, + ) + + assert route.call_count == expected_calls + assert all("outputConfig" in json.loads(call.request.content) for call in route.calls[:1]) + assert all("outputConfig" not in json.loads(call.request.content) for call in route.calls[1:]) + diff --git a/tests/unit/llms/bedrock/chat/test_native_output_fallback.py b/tests/unit/llms/bedrock/chat/test_native_output_fallback.py new file mode 100644 index 00000000000..301fdaeb245 --- /dev/null +++ b/tests/unit/llms/bedrock/chat/test_native_output_fallback.py @@ -0,0 +1,123 @@ +import asyncio +from collections.abc import Coroutine +from dataclasses import dataclass +from typing import Final + +import pytest + +from litellm.llms.bedrock.chat.native_output_fallback import NativeOutputAttemptResult, call_with_native_output_fallback +from litellm.llms.bedrock.common_utils import BedrockError, NativeOutputFallback +from litellm.types.utils import ModelResponse + +_MODEL: Final = "us.anthropic.claude-opus-4-8" +_FALLBACK: Final = NativeOutputFallback( + optional_params={"tools": "json_tool_call"}, litellm_params={"model_info": {"opted_out": True}} +) +_REJECTION: Final = BedrockError( + status_code=400, + message='{"message":"The model returned the following errors: output_config.format: Extra inputs are not permitted"}', +) +_SCHEMA_REFUSAL: Final = BedrockError( + status_code=400, message='{"message":"output_config.format.schema: pattern is too complex for structured output"}' +) +_OTHER_400: Final = BedrockError(status_code=400, message='{"message":"messages.0.content: Field required"}') + + +@dataclass(slots=True) +class ScriptedBedrock: + errors: tuple[BedrockError | None, ...] + asynchronous: bool + sent: tuple[NativeOutputFallback | None, ...] = () + + def _answer(self, error: BedrockError | None) -> ModelResponse: + if error is not None: + raise error + return ModelResponse(id=f"answer-{len(self.sent)}") + + async def _answer_later(self, error: BedrockError | None) -> ModelResponse: + await asyncio.sleep(0) + return self._answer(error) + + def __call__(self, fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult: + self.sent = (*self.sent, fallback) + error: Final = self.errors[len(self.sent) - 1] + return self._answer_later(error) if self.asynchronous else self._answer(error) + + +def _resolve(result: NativeOutputAttemptResult) -> object: + if isinstance(result, Coroutine): + return asyncio.run(result) + return result + + +def _call(bedrock: ScriptedBedrock, fallback: NativeOutputFallback | None) -> object: + return _resolve(call_with_native_output_fallback(bedrock, None if fallback is None else lambda: fallback, _MODEL)) + + +_MODES: Final = pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) + + +@_MODES +def test_a_native_output_rejection_is_sent_again_once_with_the_fallback_params(asynchronous: bool) -> None: + bedrock: Final = ScriptedBedrock(errors=(_REJECTION, None), asynchronous=asynchronous) + + response: Final = _call(bedrock, _FALLBACK) + + assert isinstance(response, ModelResponse) and response.id == "answer-2" + assert bedrock.sent == (None, _FALLBACK) + + +@_MODES +def test_a_second_native_output_rejection_reaches_the_caller(asynchronous: bool) -> None: + bedrock: Final = ScriptedBedrock(errors=(_REJECTION, _REJECTION), asynchronous=asynchronous) + + with pytest.raises(BedrockError): + _call(bedrock, _FALLBACK) + + assert len(bedrock.sent) == 2 + + +@_MODES +@pytest.mark.parametrize( + ("error", "fallback"), + [(_OTHER_400, _FALLBACK), (_REJECTION, None)], + ids=["other_400", "no_fallback"], +) +def test_no_second_request_without_both_a_rejection_and_a_fallback( + asynchronous: bool, error: BedrockError, fallback: NativeOutputFallback | None +) -> None: + bedrock: Final = ScriptedBedrock(errors=(error,), asynchronous=asynchronous) + + with pytest.raises(BedrockError) as raised: + _call(bedrock, fallback) + + assert raised.value is error + assert bedrock.sent == (None,) + + +@_MODES +def test_a_rejection_reaches_the_caller_when_the_retry_would_send_native_output_again(asynchronous: bool) -> None: + bedrock: Final = ScriptedBedrock(errors=(_REJECTION,), asynchronous=asynchronous) + + with pytest.raises(BedrockError) as raised: + _resolve(call_with_native_output_fallback(bedrock, lambda: None, _MODEL)) + + assert raised.value is _REJECTION + assert bedrock.sent == (None,) + + +@pytest.mark.parametrize( + ("error", "advises_opting_out"), + [(_REJECTION, True), (_SCHEMA_REFUSAL, False)], + ids=["native_output_unsupported", "schema_refused"], +) +def test_a_rejection_logs_the_retry_and_advises_opting_out_only_when_native_output_is_unsupported( + error: BedrockError, advises_opting_out: bool, caplog: pytest.LogCaptureFixture +) -> None: + bedrock: Final = ScriptedBedrock(errors=(error, None), asynchronous=False) + + with caplog.at_level("WARNING", logger="LiteLLM"): + _call(bedrock, _FALLBACK) + + assert _MODEL in caplog.text + assert ("supports_bedrock_native_structured_output: false" in caplog.text) is advises_opting_out diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index 6685deccf65..7ea6246025b 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -2024,6 +2024,31 @@ class TestBedrockBatchAnthropicRowParams: assert "response_format" not in model_input assert ("output_config" in model_input) == (response_format["type"] == "json_schema") + @pytest.mark.parametrize( + ("model", "native_output"), + [("bedrock/us.anthropic.claude-opus-4-8", False), (MODEL, True)], + ids=["opt_in_native_output_model", "native_output_model"], + ) + def test_thinking_json_schema_rows_use_native_output_only_on_models_that_always_take_it( + self, local_model_cost_map, model, native_output + ): + model_input: Final = self._transform( + "/v1/chat/completions", + { + "messages": [{"role": "user", "content": "Weather in Paris?"}], + "thinking": {"type": "adaptive"}, + "response_format": { + "type": "json_schema", + "json_schema": {"name": "weather", "schema": self.PARAMETERS}, + }, + }, + model=model, + ) + tool_names: Final = frozenset(tool["name"] for tool in model_input.get("tools", [])) + + assert ("format" in model_input.get("output_config", {})) is native_output + assert (tool_names == {"json_tool_call"}) is not native_output + def test_provider_native_params_still_pass_through(self): model_input = self._transform( "/v1/chat/completions", diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index 62164781271..5acedfeb9b9 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -6,6 +6,7 @@ import os import struct import zlib from collections.abc import AsyncIterator, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime from types import SimpleNamespace from typing import Final @@ -3783,3 +3784,162 @@ def test_bedrock_removed_tool_change_does_not_add_beta(explicit_beta: bool) -> N assert result["messages"] == [{"role": "user", "content": "Reply with OK"}] assert result.get("anthropic_beta", []).count(beta) == int(explicit_beta) + + +_NATIVE_OUTPUT_REJECTION: Final = { + "message": "The model returned the following errors: compilation exceeded limits: Schema is too complex." +} +_STALE_SIGNATURE_ERROR: Final = {"message": "messages.1.content.0: Invalid `signature` in `thinking` block"} +_OPTED_OUT: Final = {"supports_bedrock_native_structured_output": False} +_CONSTRAINED_SCHEMA: Final = { + "type": "object", + "properties": { + "summary": {"type": "string", "maxLength": 50}, + "kind": {"oneOf": [{"type": "string"}, {"type": "integer"}]}, + }, + "required": ["summary", "kind"], + "additionalProperties": False, +} +_ANTHROPIC_MESSAGE: Final = { + "id": "msg_scripted", + "type": "message", + "role": "assistant", + "model": "claude-scripted", + "content": [{"type": "text", "text": '{"summary": "sunny", "kind": 1}'}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, +} +_THINKING_HISTORY: Final = ( + {"role": "user", "content": "Describe the weather."}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "checking", "signature": "stale"}, + {"type": "text", "text": "Sunny."}, + ], + }, + {"role": "user", "content": "Summarise it as JSON."}, +) + + +@dataclass(slots=True) +class ScriptedBedrockInvoke: + replies: tuple[tuple[int, Mapping[str, object]], ...] + bodies: tuple[dict[str, object], ...] = () + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.bodies = (*self.bodies, REASONING_BODY_ADAPTER.validate_json(request.content)) + status_code, body = self.replies[len(self.bodies) - 1] + return httpx.Response(status_code, json=body) + + +async def _send_bedrock_messages( + model: str, + model_info: Mapping[str, object] | None, + replies: Sequence[tuple[int, Mapping[str, object]]], + messages: Sequence[Mapping[str, object]] = ({"role": "user", "content": "Summarise the weather as JSON."},), +) -> tuple[ScriptedBedrockInvoke, Exception | None]: + bedrock: Final = ScriptedBedrockInvoke(replies=tuple(replies)) + try: + await litellm.anthropic_messages( + model=f"bedrock/invoke/{model}", + messages=[dict(message) for message in messages], + max_tokens=4000, + system="Answer in French.", + thinking={"type": "adaptive"}, + output_config={"format": {"type": "json_schema", "schema": _CONSTRAINED_SCHEMA}}, + aws_access_key_id="test-access-key", + aws_secret_access_key="test-secret-key", + aws_region_name="us-east-1", + client=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock)), + **({} if model_info is None else {"model_info": model_info}), + ) + except Exception as error: + return bedrock, error + return bedrock, None + + +_OPT_IN_DEPLOYMENTS: Final = pytest.mark.parametrize( + ("model", "model_info"), + [ + ("us.anthropic.claude-opus-4-8", None), + ("us.anthropic.claude-opus-5-5", {"supports_bedrock_native_structured_output": True}), + ], + ids=["mapped_opt_in", "deployment_opt_in"], +) + + +@pytest.mark.usefixtures("local_model_cost_map") +@_OPT_IN_DEPLOYMENTS +async def test_bedrock_messages_native_output_rejection_resends_the_body_an_opted_out_deployment_sends( + model: str, model_info: Mapping[str, object] | None +) -> None: + opt_in, opt_in_error = await _send_bedrock_messages( + model, model_info, [(400, _NATIVE_OUTPUT_REJECTION), (200, _ANTHROPIC_MESSAGE)] + ) + opted_out, opted_out_error = await _send_bedrock_messages(model, _OPTED_OUT, [(200, _ANTHROPIC_MESSAGE)]) + + assert opt_in_error is None and opted_out_error is None + native, retried = opt_in.bodies + assert "format" in REASONING_BODY_ADAPTER.validate_python(native["output_config"]) + assert opted_out.bodies == (retried,) + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("model_info", [None, _OPTED_OUT], ids=["native_output", "opted_out"]) +async def test_bedrock_messages_thinking_signature_400_that_persists_reaches_bedrock_twice_as_before( + model_info: Mapping[str, object] | None, +) -> None: + bedrock, error = await _send_bedrock_messages( + "us.anthropic.claude-opus-4-8", model_info, [(400, _STALE_SIGNATURE_ERROR)] * 3, _THINKING_HISTORY + ) + + assert isinstance(error, litellm.BadRequestError) + assert len(bedrock.bodies) == 2 + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + "first_error, second_error", + [(_NATIVE_OUTPUT_REJECTION, _STALE_SIGNATURE_ERROR), (_STALE_SIGNATURE_ERROR, _NATIVE_OUTPUT_REJECTION)], + ids=["rejection_first", "signature_first"], +) +async def test_bedrock_messages_native_output_rejection_and_signature_400_each_get_one_retry_like_an_opted_out_deployment( + first_error: Mapping[str, object], second_error: Mapping[str, object] +) -> None: + opt_in, opt_in_error = await _send_bedrock_messages( + "us.anthropic.claude-opus-4-8", + None, + [(400, first_error), (400, second_error), (200, _ANTHROPIC_MESSAGE)], + _THINKING_HISTORY, + ) + opted_out, opted_out_error = await _send_bedrock_messages( + "us.anthropic.claude-opus-4-8", + _OPTED_OUT, + [(400, _STALE_SIGNATURE_ERROR), (200, _ANTHROPIC_MESSAGE)], + _THINKING_HISTORY, + ) + + assert opt_in_error is None and opted_out_error is None + assert len(opt_in.bodies) == 3 + assert opt_in.bodies[-1] == opted_out.bodies[-1] + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + ("model", "model_info", "error"), + [ + ("us.anthropic.claude-opus-4-8", None, {"message": "messages.0.content: Field required"}), + ("us.anthropic.claude-sonnet-4-6", None, _NATIVE_OUTPUT_REJECTION), + ("us.anthropic.claude-opus-4-8", _OPTED_OUT, _NATIVE_OUTPUT_REJECTION), + ], + ids=["other_400", "native_model", "deployment_opt_out"], +) +async def test_bedrock_messages_400_is_not_retried_unless_it_rejects_opt_in_native_output( + model: str, model_info: Mapping[str, object] | None, error: Mapping[str, object] +) -> None: + bedrock, raised = await _send_bedrock_messages(model, model_info, [(400, error), (200, _ANTHROPIC_MESSAGE)]) + + assert isinstance(raised, litellm.BadRequestError) + assert len(bedrock.bodies) == 1 diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index 34df830a053..d283aa7dfe6 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -1,15 +1,20 @@ import asyncio, importlib, litellm, litellm.litellm_core_utils.get_model_cost_map as bedrock_govcloud_model_cost_map, pytest +from typing import Final from litellm.llms.bedrock.common_utils import( AmazonBedrockGlobalConfig, BedrockModelInfo, + bedrock_native_output_is_opt_in, + deployment_model_info, extract_model_name_from_bedrock_arn, get_bedrock_base_model, get_bedrock_cross_region_inference_regions, + is_bedrock_native_output_rejection, strip_bedrock_routing_prefix, strip_bedrock_throughput_suffix, + without_bedrock_native_output_opt_in, ) from collections.abc import Iterator from litellm import completion @@ -2011,3 +2016,75 @@ def test_event_decoder_reports_only_missing_aws_dependency(missing): assert error.value.__cause__ is failure else: assert error.value is failure + + +_FORMAT_REJECTION: Final = "The model returned the following errors: output_config.format: Extra inputs are not permitted" + + +_COMPILE_REJECTION: Final = "The model returned the following errors: compilation exceeded limits: Schema is too complex." + + +_SCHEMA_PATTERN_REJECTION: Final = ( + "The model returned the following errors: output_config.format.schema: Unsupported regex feature in pattern " + "field: pattern is too complex for structured output: reduce the {n,m} upper bound, narrow the character class " + "range, or avoid nesting quantified groups" +) + + +@pytest.mark.parametrize( + ("status_code", "error_text", "expected"), + [ + (400, f'{{"message":"{_FORMAT_REJECTION}"}}', True), + (400, f'{{"message":"{_COMPILE_REJECTION}"}}', True), + (400, f'{{"message":"{_SCHEMA_PATTERN_REJECTION}"}}', True), + (400, "Malformed input request: #: extraneous key [output_config] is not permitted", False), + (400, "The model returned the following errors: messages.0.content: Field required", False), + (500, _FORMAT_REJECTION, False), + (429, _COMPILE_REJECTION, False), + ], + ids=[ + "format_not_permitted", + "schema_too_complex", + "schema_pattern_too_complex", + "other_key_not_permitted", + "other_400", + "500", + "429", + ], +) +def test_is_bedrock_native_output_rejection_matches_only_the_400s_that_reject_native_output( + status_code, error_text, expected +): + assert is_bedrock_native_output_rejection(status_code, error_text) is expected + + +@pytest.mark.parametrize( + ("model", "model_info", "expected"), + [ + ("us.anthropic.claude-opus-4-8", None, True), + ("global.anthropic.claude-sonnet-5", None, True), + ("us-gov.anthropic.claude-opus-4-8", None, False), + ("us.anthropic.claude-opus-4-8", {"supports_bedrock_native_structured_output": False}, False), + ("us.anthropic.claude-opus-5-5", {"supports_bedrock_native_structured_output": True}, True), + ("us.anthropic.claude-sonnet-4-6", {"supports_bedrock_native_structured_output": True}, False), + ], + ids=["opus_4_8", "sonnet_5", "govcloud_opus_4_8", "deployment_opt_out", "deployment_opt_in", "native_model"], +) +def test_bedrock_native_output_is_opt_in_only_where_the_opt_in_key_turns_native_output_on( + local_model_cost_map, model, model_info, expected +): + assert bedrock_native_output_is_opt_in(model, model_info=model_info) is expected + + +def test_without_bedrock_native_output_opt_in_turns_the_opt_in_off_and_keeps_the_deployment(local_model_cost_map): + litellm_params: Final = {"api_base": "https://bedrock.example", "model_info": {"id": "deployment-1"}} + + turned_off: Final = without_bedrock_native_output_opt_in(litellm_params) + + assert not bedrock_native_output_is_opt_in( + "us.anthropic.claude-opus-4-8", model_info=deployment_model_info(turned_off) + ) + assert turned_off["api_base"] == "https://bedrock.example" + assert deployment_model_info(turned_off)["id"] == "deployment-1" + assert litellm_params["model_info"] == {"id": "deployment-1"} + diff --git a/tests/unit/types/test_completion.py b/tests/unit/types/test_completion.py index 28d13b7da4f..fbb9f0dab43 100644 --- a/tests/unit/types/test_completion.py +++ b/tests/unit/types/test_completion.py @@ -179,6 +179,7 @@ def _build_dispatch_context() -> CompletionDispatchContext: model="gpt-4o", model_response=None, # type: ignore[arg-type] optional_params={}, + optional_params_for_model_info=lambda model_info: {}, organization=None, provider_config=None, request_params={},