From 36c64ce85d1abb6db3a4c755d7c8af701e6d259e Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:38:00 -0800 Subject: [PATCH] feat(bedrock): default to native structured output on Opus 5.5, Fable 5.1, Sonnet 5 and Haiku 5.5 (#50221) * feat(bedrock): enable native structured output by default for selected Claude models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): keep govcloud and bare model rows off native output Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): retry rejected native schemas with the JSON tool on default-native models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): keep the rejection retry on the unforceable-tool native path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): retry rejected schemas on the Messages path for default-native models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(bedrock): format Messages native output retry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(bedrock): allow profile-only native output metadata Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): only disable default-native output on schema-rejection retries Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(bedrock): type fixture parameters in native output retry tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): retry with the JSON tool when Bedrock rejects a schema's grammar size Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): drop strict tools after a native-output retry hits the grammar limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(bedrock): format native output fallback Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): honor the deployment native output opt-out and keep wide batch schemas on the JSON tool A deployment's supports_bedrock_native_structured_output: false now keeps the JSON tool on rows that default to native output. Bedrock batch has no retry, so a record whose schema has more than 64 properties keeps the JSON tool there too. * fix(bedrock): retry a native structured output stream as a buffered stream when Bedrock denies the stream action A role allowed only bedrock:InvokeModel got a 403 on schema streams once these models went native, since native streams call converse-stream while the JSON tool path buffered through converse. The retry now sends that request again as a buffered JSON tool stream, the shape the base sent. The wire test moves its neither-key and opt-in cells to Opus 4.7, its Bedrock-key-only cells to Sonnet 5.5, and its unforceable-tool cells to the bare Opus 5.5 id, since this PR makes Opus 5.5 and Haiku 5.5 always native. The batch wire test now checks a thinking schema record on both sides of the contract the batch unit test already pins, the JSON tool on opt-in Sonnet 5.5 and native output on always-native Haiku 5.5, since its Haiku 5.5 and Opus 4.8 cells assumed those models were opt-in. * fix(bedrock): move the Claude 5 native output default to a key released proxies do not read Released proxies load the cost map from main and have none of the native output retries, so setting supports_native_structured_output on the Opus 5.5, Fable 5.1, Sonnet 5 and Haiku 5.5 profile rows turned working wide or min/max schemas into 400s on v1.108.1. Those rows keep supports_native_structured_output false and gain bedrock_native_structured_output_by_default true, which get_model_info reports as supports_native_structured_output, so this version behaves as before while released versions keep the JSON tool. * fix(bedrock): rename the Claude 5 native output default key to supports_bedrock_native_structured_output_by_default The cost map guard runs main's model catalog on the PR's cost map, and that catalog rejects any undeclared key that does not start with supports_. Declaring the old name on main first needs its own PR, so the key takes the supports_ prefix. Released proxies still never read it * fix(bedrock): honor registered and batch opt-outs, move Opus 5 and 4.8 to the default key A registered supports_bedrock_native_structured_output false now beats the native output default, and batch uploads read the deployment's model_info, so an opted-out deployment keeps the JSON tool on every record. The Opus 5 and Opus 4.8 profile rows move to supports_bedrock_native_structured_output_by_default, so released proxies on main's cost map keep the JSON tool on them. The 64 property batch cutoff now applies only to rows with that key, so models that were already native keep their batch output shape. * fix(proxy): report a deployment's Bedrock native output opt-out on /model/info /model/info merged the deployment's model_info over the backend row, so a deployment that set only supports_bedrock_native_structured_output: false still showed the backend's derived supports_native_structured_output: true. The generic flag is now derived from the merged info unless the deployment set it itself, through the same helper get_model_info uses. * fix(bedrock): keep native output when a denied schema stream is resent buffered A role without bedrock:InvokeModelWithResponseStream had its native output schema stream resent as a buffered JSON tool call. The resend now keeps the native params with fake_stream set, and falls back to the JSON tool only if Bedrock then rejects native output. * fix(bedrock): drop rejected strict tools on the JSON tool retry after a buffered native resend * fix(bedrock): send a buffered schema stream to /invoke and keep the by-default key on its deployment The shared HTTP handler now builds the non-streaming URL for a fake-streamed request, so the invoke route's buffered resend after a denied stream calls /invoke instead of invoke-with-response-stream. supports_bedrock_native_structured_output_by_default is deployment scoped, so setting it on one deployment no longer reaches sibling deployments of the same model * fix(bedrock): answer a buffered invoke stream from the non-streaming reply * fix(bedrock): keep the JSON tool retry buffered after a denied stream's schema is rejected * fix(bedrock): count only a true fake_stream and keep the deployment by-default key out of /model/info A fake_stream sent as "true" or 1 picked the non-streaming URL but kept the streaming reader, so Bedrock invoke streams answered 500 after three billed calls. Only a real True counts now, as on main. /model/info read supports_bedrock_native_structured_output_by_default from the deployment, while routing reads it only from the cost map, so it reported native output for deployments that use the JSON tool. --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../bedrock/chat/converse_transformation.py | 3 +- .../bedrock/chat/native_output_fallback.py | 157 +++++++++++++---- litellm/llms/bedrock/common_utils.py | 37 +++- litellm/llms/bedrock/files/transformation.py | 48 +++++- litellm/llms/custom_httpx/llm_http_handler.py | 8 +- litellm/main.py | 33 +++- ...odel_prices_and_context_window_backup.json | 57 +++++-- litellm/proxy/proxy_server.py | 21 ++- litellm/types/utils.py | 2 + litellm/utils.py | 12 +- model_prices_and_context_window.json | 57 +++++-- .../test_bedrock_batch_files_wire.py | 18 +- ...t_bedrock_native_structured_output_wire.py | 15 +- tests/unit/conftest.py | 5 +- ...mations_anthropic_claude_transformation.py | 14 +- .../chat/test_bedrock_converse_handler.py | 133 +++++++++++++++ .../chat/test_converse_transformation.py | 109 +++++++++--- .../chat/test_native_output_fallback.py | 116 +++++++++++++ .../test_bedrock_files_transformation.py | 72 +++++++- .../llms/bedrock/test_bedrock_common_utils.py | 158 +++++++----------- .../custom_httpx/test_llm_http_handler.py | 86 ++++++++++ .../proxy_server/test_routes_model_info.py | 45 +++++ .../unit/test_router_model_cost_isolation.py | 8 +- tests/unit/test_utils.py | 1 + tests/unit/test_utils_get_model_info.py | 25 ++- 25 files changed, 1024 insertions(+), 216 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index b93d40738ea..12c5031628b 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -960,7 +960,8 @@ class AmazonConverseConfig(BaseConfig): ) -> bool: """Check if the Bedrock model takes native structured outputs (outputConfig.textFormat) for this request. - ``supports_native_structured_output`` in ``litellm.model_cost`` turns it on for every request, and + ``supports_native_structured_output`` or ``supports_bedrock_native_structured_output_by_default`` in + ``litellm.model_cost`` turns it on for every request, and ``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) also when Claude will think, which the json tool path would drop. When the request does not ask for thinking, a model that cannot force the json tool gets it unless that key is false. diff --git a/litellm/llms/bedrock/chat/native_output_fallback.py b/litellm/llms/bedrock/chat/native_output_fallback.py index 79742d0810c..7d5662e2024 100644 --- a/litellm/llms/bedrock/chat/native_output_fallback.py +++ b/litellm/llms/bedrock/chat/native_output_fallback.py @@ -9,7 +9,9 @@ from litellm.llms.bedrock.common_utils import ( NativeOutputFallback, bedrock_strict_tools_fallback, is_bedrock_native_output_rejection, + is_bedrock_stream_permission_denial, warn_bedrock_native_output_rejected, + warn_bedrock_native_output_stream_denied, warn_bedrock_strict_tools_dropped, ) from litellm.types.utils import ModelResponse @@ -25,22 +27,55 @@ NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | Non class _SelectedFallback: fallback: NativeOutputFallback native_output_retry: bool + buffered_native_output: bool = False _RetryFor: TypeAlias = Callable[[BaseLLMException], _SelectedFallback | None] +_NextRetryFor: TypeAlias = Callable[[BaseLLMException, _SelectedFallback], _SelectedFallback | None] -def _native_output_retry( - error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str +def _native_output_rejection_retry( + error: BaseLLMException, fallback: NativeOutputFallbackFactory, model: str ) -> NativeOutputFallback | None: - if fallback is None or not is_bedrock_native_output_rejection(error.status_code, error.message): - return None retry: Final = fallback() if retry is not None: warn_bedrock_native_output_rejected(model, error.message) return retry +def _buffered(retry: NativeOutputFallback) -> NativeOutputFallback: + return NativeOutputFallback( + optional_params={**retry.optional_params, "fake_stream": True}, litellm_params=retry.litellm_params + ) + + +def _buffered_stream_retry(fallback: NativeOutputFallbackFactory, model: str) -> NativeOutputFallback | None: + retry: Final = fallback() + if retry is None or retry.optional_params.get("fake_stream") is not True: + return None + warn_bedrock_native_output_stream_denied(model) + return retry + + +def _native_output_retry( + error: BaseLLMException, + fallback: NativeOutputFallbackFactory | None, + buffered_stream_fallback: NativeOutputFallbackFactory | None, + model: str, +) -> _SelectedFallback | None: + if fallback is not None and is_bedrock_native_output_rejection(error.status_code, error.message): + rejection_retry: Final = _native_output_rejection_retry(error, fallback, model) + return None if rejection_retry is None else _SelectedFallback(rejection_retry, native_output_retry=True) + if buffered_stream_fallback is not None and is_bedrock_stream_permission_denial(error.status_code, error.message): + stream_retry: Final = _buffered_stream_retry(buffered_stream_fallback, model) + return ( + None + if stream_retry is None + else _SelectedFallback(stream_retry, native_output_retry=True, buffered_native_output=True) + ) + return None + + def _strict_tools_retry( error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str ) -> NativeOutputFallback | None: @@ -55,12 +90,13 @@ def _strict_tools_retry( def _fallback_for( error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, + buffered_stream_fallback: NativeOutputFallbackFactory | None, strict_tools_fallback: NativeOutputFallbackFactory | None, model: str, ) -> _SelectedFallback | None: - native_output_retry: Final = _native_output_retry(error, fallback, model) + native_output_retry: Final = _native_output_retry(error, fallback, buffered_stream_fallback, model) if native_output_retry is not None: - return _SelectedFallback(native_output_retry, native_output_retry=True) + return native_output_retry strict_tools_retry: Final = _strict_tools_retry(error, strict_tools_fallback, model) if strict_tools_retry is None: return None @@ -87,12 +123,81 @@ def _strict_tools_retry_after_native_output_retry( return composed_retry +def _next_retry( + error: BaseLLMException, + retry: _SelectedFallback, + fallback: NativeOutputFallbackFactory | None, + strict_tools_fallback: NativeOutputFallbackFactory | None, + model: str, +) -> _SelectedFallback | None: + if ( + retry.buffered_native_output + and fallback is not None + and is_bedrock_native_output_rejection(error.status_code, error.message) + ): + rejection_retry: Final = _native_output_rejection_retry(error, fallback, model) + return ( + None if rejection_retry is None else _SelectedFallback(_buffered(rejection_retry), native_output_retry=True) + ) + composed_retry: Final = _strict_tools_retry_after_native_output_retry(error, retry, strict_tools_fallback, model) + return None if composed_retry is None else _SelectedFallback(composed_retry, native_output_retry=False) + + +def _attempt_or_next_retry( + attempt: NativeOutputAttempt, retry: _SelectedFallback, next_retry_for: _NextRetryFor +) -> NativeOutputAttemptResult | _SelectedFallback: + try: + return attempt(retry.fallback) + except BaseLLMException as error: + next_retry: Final = next_retry_for(error, retry) + if next_retry is None: + raise + return next_retry + + +def _attempt_retries( + attempt: NativeOutputAttempt, retry: _SelectedFallback, next_retry_for: _NextRetryFor +) -> NativeOutputAttemptResult: + after_first_retry: Final = _attempt_or_next_retry(attempt, retry, next_retry_for) + if not isinstance(after_first_retry, _SelectedFallback): + return after_first_retry + after_second_retry: Final = _attempt_or_next_retry(attempt, after_first_retry, next_retry_for) + if not isinstance(after_second_retry, _SelectedFallback): + return after_second_retry + return attempt(after_second_retry.fallback) + + +async def _await_or_next_retry( + attempt: NativeOutputAttempt, retry: _SelectedFallback, next_retry_for: _NextRetryFor +) -> NativeOutputResponse | _SelectedFallback: + try: + result: Final = attempt(retry.fallback) + return await result if isinstance(result, Coroutine) else result + except BaseLLMException as error: + next_retry: Final = next_retry_for(error, retry) + if next_retry is None: + raise + return next_retry + + +async def _await_retries( + attempt: NativeOutputAttempt, retry: _SelectedFallback, next_retry_for: _NextRetryFor +) -> NativeOutputResponse: + after_first_retry: Final = await _await_or_next_retry(attempt, retry, next_retry_for) + if not isinstance(after_first_retry, _SelectedFallback): + return after_first_retry + after_second_retry: Final = await _await_or_next_retry(attempt, after_first_retry, next_retry_for) + if not isinstance(after_second_retry, _SelectedFallback): + return after_second_retry + last_attempt: Final = attempt(after_second_retry.fallback) + return await last_attempt if isinstance(last_attempt, Coroutine) else last_attempt + + async def _await_with_native_output_fallback( first_attempt: Coroutine[object, object, NativeOutputResponse], attempt: NativeOutputAttempt, retry_for: _RetryFor, - strict_tools_fallback: NativeOutputFallbackFactory | None, - model: str, + next_retry_for: _NextRetryFor, ) -> NativeOutputResponse: try: return await first_attempt @@ -100,17 +205,7 @@ async def _await_with_native_output_fallback( retry: Final = retry_for(error) if retry is None: raise - try: - second_attempt: Final = attempt(retry.fallback) - return await second_attempt if isinstance(second_attempt, Coroutine) else second_attempt - except BaseLLMException as error: - composed_retry: Final = _strict_tools_retry_after_native_output_retry( - error, retry, strict_tools_fallback, model - ) - if composed_retry is None: - raise - third_attempt: Final = attempt(composed_retry) - return await third_attempt if isinstance(third_attempt, Coroutine) else third_attempt + return await _await_retries(attempt, retry, next_retry_for) def call_with_native_output_fallback( @@ -118,11 +213,19 @@ def call_with_native_output_fallback( fallback: NativeOutputFallbackFactory | None, model: str, strict_tools_fallback: NativeOutputFallbackFactory | None = None, + buffered_stream_fallback: NativeOutputFallbackFactory | None = None, ) -> NativeOutputAttemptResult: - if fallback is None and strict_tools_fallback is None: + if fallback is None and strict_tools_fallback is None and buffered_stream_fallback is None: return attempt(None) retry_for: Final = partial( - _fallback_for, fallback=fallback, strict_tools_fallback=strict_tools_fallback, model=model + _fallback_for, + fallback=fallback, + buffered_stream_fallback=buffered_stream_fallback, + strict_tools_fallback=strict_tools_fallback, + model=model, + ) + next_retry_for: Final = partial( + _next_retry, fallback=fallback, strict_tools_fallback=strict_tools_fallback, model=model ) try: first_attempt: Final = attempt(None) @@ -130,15 +233,7 @@ def call_with_native_output_fallback( retry: Final = retry_for(error) if retry is None: raise - try: - return attempt(retry.fallback) - except BaseLLMException as retry_error: - composed_retry: Final = _strict_tools_retry_after_native_output_retry( - retry_error, retry, strict_tools_fallback, model - ) - if composed_retry is None: - raise - return attempt(composed_retry) + return _attempt_retries(attempt, retry, next_retry_for) if isinstance(first_attempt, Coroutine): - return _await_with_native_output_fallback(first_attempt, attempt, retry_for, strict_tools_fallback, model) + return _await_with_native_output_fallback(first_attempt, attempt, retry_for, next_retry_for) return first_attempt diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index f0627c7362c..0f8694a7e46 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -297,9 +297,16 @@ def _bedrock_model_supports(model: str, key: str, custom_llm_provider: str = "be def _bedrock_native_output_enabled( model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None ) -> bool: - deployment_flag: Final = (model_info or {}).get("supports_native_structured_output") + deployment_info: Final = model_info or {} + deployment_flag: Final = deployment_info.get("supports_native_structured_output") if isinstance(deployment_flag, bool): return deployment_flag + return deployment_info.get("supports_bedrock_native_structured_output") is not False and ( + bedrock_model_always_takes_native_output(model, custom_llm_provider) + ) + + +def bedrock_model_always_takes_native_output(model: str, custom_llm_provider: str = "bedrock") -> bool: return _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) @@ -443,6 +450,18 @@ def warn_bedrock_native_output_rejected(model: str, error_text: str) -> None: ) +def is_bedrock_stream_permission_denial(status_code: int, error_text: str) -> bool: + return status_code == 403 and "bedrock:InvokeModelWithResponseStream" in error_text + + +def warn_bedrock_native_output_stream_denied(model: str) -> None: + verbose_logger.warning( + "Bedrock denied bedrock:InvokeModelWithResponseStream for model=%s, so litellm sent the structured output " + "request again as a buffered stream. Allow that action to stream it.", + model, + ) + + @dataclass(frozen=True, slots=True) class NativeOutputFallback: optional_params: Mapping[str, object] @@ -543,6 +562,22 @@ def json_object_or_none(value: object) -> Mapping[str, object] | None: BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES: Final = 64 +def _response_format_json_schema(response_format: object) -> Mapping[str, object] | None: + json_schema: Final = json_object_or_none((json_object_or_none(response_format) or {}).get("json_schema")) + return json_object_or_none((json_schema or {}).get("schema")) + + +def bedrock_batch_record_takes_native_output( + model: str, response_format: object, model_info: Mapping[str, object] | None = None +) -> bool: + schema: Final = _response_format_json_schema(response_format) + return _bedrock_native_output_enabled(model, "bedrock", model_info) and ( + schema is None + or schema_property_count(schema) <= BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES + or not _bedrock_model_supports(model, "supports_bedrock_native_structured_output_by_default") + ) + + def bedrock_cleaned_output_schema( model: str, schema: Mapping[str, object], custom_llm_provider: str = "bedrock" ) -> Mapping[str, object] | None: diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 4d137876890..ef54c85c131 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -71,6 +71,7 @@ from litellm.utils import get_llm_provider, get_optional_params from ..base_aws_llm import BaseAWSLLM from ..common_utils import ( BedrockError, + bedrock_batch_record_takes_native_output, deployment_model_info, merge_bedrock_aws_request_params, resolve_s3_encryption_key_id, @@ -167,6 +168,20 @@ _LITELLM_PARAMS_THE_MAPPER_TAKES: Final = frozenset({"allowed_openai_params"}) _MAPPED_PARAMS_THE_REQUEST_HANDLER_STRIPS: Final = frozenset({"json_mode"}) +_UPLOAD_PARAMS: Final = TypeAdapter(dict[str, object]) + + +def _upload_params(params: object) -> Mapping[str, object]: + return _UPLOAD_PARAMS.validate_python(params) if isinstance(params, Mapping) else {} + + +def _upload_deployment_model_info(litellm_params: object) -> Mapping[str, object]: + upload_params: Final = _upload_params(litellm_params) + return deployment_model_info(upload_params) or deployment_model_info( + _upload_params(upload_params.get("litellm_metadata")) + ) + + def _invoke_route_model(model: str) -> str: return f"invoke/{_strip_llm_routing_prefix(model).removeprefix('invoke/')}" @@ -1034,6 +1049,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): openai_request_body: Mapping[str, Any], model: str, provider: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> dict[str, object]: """ Transform OpenAI request body to Bedrock-compatible modelInput @@ -1055,7 +1071,15 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): ) config: Final = AmazonAnthropicClaudeConfig() - litellm_params: Final = without_bedrock_native_output_opt_in({}) + litellm_params: Final = without_bedrock_native_output_opt_in( + { + "model_info": { + "supports_native_structured_output": bedrock_batch_record_takes_native_output( + _invoke_route_model(model), optional_params.get("response_format"), model_info + ) + } + } + ) mapped_params = get_optional_params( model=_invoke_route_model(model), custom_llm_provider="bedrock", @@ -1150,25 +1174,32 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): ] def transform_batch_input_lines( - self, batch_input_lines: Sequence[_BatchInputLine], target_model: str = "" + self, + batch_input_lines: Sequence[_BatchInputLine], + target_model: str = "", + model_info: Mapping[str, object] | None = None, ) -> tuple[_BedrockBatchRecord, ...]: return tuple( - self._transform_batch_input_line(idx, batch_input_line, target_model) + self._transform_batch_input_line(idx, batch_input_line, target_model, model_info) for idx, batch_input_line in enumerate(batch_input_lines) ) def _transform_batch_input_line( - self, idx: int, batch_input_line: _BatchInputLine, target_model: str + self, idx: int, batch_input_line: _BatchInputLine, target_model: str, model_info: Mapping[str, object] | None ) -> _BedrockBatchRecord: try: - return self._transform_openai_batch_record(idx, batch_input_line.record, target_model) + return self._transform_openai_batch_record(idx, batch_input_line.record, target_model, model_info) except _BATCH_RECORD_CONVERSION_ERRORS as e: raise _batch_line_error( batch_input_line.line_number, f"could not be converted to a Bedrock batch record: {e}" ) from e def _transform_openai_batch_record( - self, idx: int, openai_record: _OpenAIBatchRecord, target_model: str + self, + idx: int, + openai_record: _OpenAIBatchRecord, + target_model: str, + model_info: Mapping[str, object] | None = None, ) -> _BedrockBatchRecord: import litellm @@ -1192,6 +1223,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): openai_request_body=self._transform_batch_body_to_chat_body(openai_body, record_kind), model=model_for_transform, provider=provider, + model_info=model_info, ) ) return _BedrockBatchRecord( @@ -1211,7 +1243,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): model: str, create_file_data: CreateFileRequest, optional_params: dict, - litellm_params: dict, + litellm_params: dict[str, object], ) -> bytes | str | dict: """ Transform file request and return a pre-signed request for S3. @@ -1237,7 +1269,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): litellm_params_model: Final = litellm_params.get("model") target_model: Final = model or (litellm_params_model if isinstance(litellm_params_model, str) else "") bedrock_jsonl_content: Final = self.transform_batch_input_lines( - batch_input_lines, target_model=target_model + batch_input_lines, target_model=target_model, model_info=_upload_deployment_model_info(litellm_params) ) file_content = "\n".join(json.dumps(item) for item in bedrock_jsonl_content) elif isinstance(extracted_file_data_content, bytes): diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 169c21cf635..df82891f789 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -897,7 +897,7 @@ class BaseLLMHTTPHandler: fake_stream = ( fake_stream - or optional_params.pop("fake_stream", False) + or optional_params.pop("fake_stream", False) is True or provider_config.should_fake_stream(model=model, custom_llm_provider=custom_llm_provider, stream=stream) ) @@ -920,7 +920,7 @@ class BaseLLMHTTPHandler: api_key=api_key, model=model, optional_params=optional_params, - stream=stream, + stream=False if fake_stream else stream, litellm_params=litellm_params, ) @@ -1056,7 +1056,7 @@ class BaseLLMHTTPHandler: provider_config=provider_config, fake_stream=fake_stream, ) - if provider_config.has_custom_stream_wrapper is True: + if provider_config.has_custom_stream_wrapper is True and not fake_stream: return provider_config.get_sync_custom_stream_wrapper( model=model, custom_llm_provider=custom_llm_provider, @@ -1225,7 +1225,7 @@ class BaseLLMHTTPHandler: json_mode: bool | None = None, signed_json_body: bytes | None = None, ): - if provider_config.has_custom_stream_wrapper is True: + if provider_config.has_custom_stream_wrapper is True and not fake_stream: return await provider_config.get_async_custom_stream_wrapper( model=model, custom_llm_provider=custom_llm_provider, diff --git a/litellm/main.py b/litellm/main.py index 6b9964a450f..bdb35823757 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4393,6 +4393,14 @@ def _converse_native_output_fallback( return None if AmazonConverseConfig.sends_native_text_format(fallback.optional_params) else fallback +def _buffered_stream_fallback( + first_attempt_params: Mapping[str, object], first_attempt_litellm_params: Mapping[str, object] +) -> NativeOutputFallback: + return NativeOutputFallback( + optional_params={**first_attempt_params, "fake_stream": True}, litellm_params=first_attempt_litellm_params + ) + + def _native_output_attempt_timeout( ctx: CompletionDispatchContext, fallback: NativeOutputFallback | None ) -> float | str | httpx.Timeout | None: @@ -4496,11 +4504,14 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu ) converse_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params) + converse_uses_native_output: Final = AmazonConverseConfig().uses_native_output( + converse_model, converse_params, litellm_params + ) response = call_with_native_output_fallback( converse_attempt, ( partial(_converse_native_output_fallback, ctx, converse_params, dict(litellm_params)) - if AmazonConverseConfig().uses_native_output(converse_model, converse_params, litellm_params) + if converse_uses_native_output else None ), converse_model, @@ -4509,6 +4520,11 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu if drops_unsupported_params(litellm_params.get("drop_params")) else None ), + buffered_stream_fallback=( + partial(_buffered_stream_fallback, converse_params, dict(litellm_params)) + if converse_uses_native_output + else None + ), ) elif bedrock_route == "converse_like": model = model.replace("converse_like/", "") @@ -4551,15 +4567,20 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu ) invoke_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params) + invoke_uses_native_output: Final = isinstance( + provider_config, AmazonAnthropicClaudeConfig + ) and provider_config.uses_native_output(model, invoke_params, litellm_params) response = call_with_native_output_fallback( invoke_attempt, - ( - partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params)) - if isinstance(provider_config, AmazonAnthropicClaudeConfig) - and provider_config.uses_native_output(model, invoke_params, litellm_params) + partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params)) + if invoke_uses_native_output + else None, + model, + buffered_stream_fallback=( + partial(_buffered_stream_fallback, invoke_params, dict(litellm_params)) + if invoke_uses_native_output else None ), - model, ) return response diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4e81afb0ea6..d9d996208d8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1961,6 +1961,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2041,6 +2042,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2116,6 +2118,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2242,7 +2245,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2287,6 +2291,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2331,7 +2336,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2376,6 +2382,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2420,7 +2427,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2465,6 +2473,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2504,7 +2513,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2549,6 +2559,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2593,7 +2604,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2633,6 +2645,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2715,7 +2728,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2759,7 +2773,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2803,7 +2818,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2847,7 +2863,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2891,7 +2908,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3023,6 +3041,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3068,6 +3087,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3113,6 +3133,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3158,6 +3179,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3203,6 +3225,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -75549,7 +75572,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -75589,7 +75613,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -75629,6 +75654,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -75702,6 +75728,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77417,6 +77444,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77474,6 +77502,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77651,6 +77680,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77703,6 +77733,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77750,6 +77781,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77863,6 +77895,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f5369bc34b0..7f48d77ff09 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -221,7 +221,12 @@ from litellm.types.utils import ( is_server_derived_pricing_key, pricing_override_fields, ) -from litellm.utils import acredential_updates, cost_map_omits_token_price, with_named_credential +from litellm.utils import ( + acredential_updates, + cost_map_omits_token_price, + native_structured_output_support, + with_named_credential, +) if TYPE_CHECKING: from aiohttp import ClientSession @@ -16733,7 +16738,7 @@ def _enrich_model_info_with_litellm_data( stamped_model_info: Final = MappingProxyType( {**model_info, **_pricing_override_stamps(model_info, model.get("litellm_params") or MappingProxyType({}))} ) - model["model_info"] = { + merged_model_info: Final = { **stamped_model_info, **{ k: None if unpriced and k in ("input_cost_per_token", "output_cost_per_token") else v @@ -16741,6 +16746,18 @@ def _enrich_model_info_with_litellm_data( if k not in stamped_model_info or (stamped_model_info[k] is None and k in discovered_model_info) }, } + native_output_info: Final = { + **merged_model_info, + "supports_bedrock_native_structured_output_by_default": litellm_model_info.get( + "supports_bedrock_native_structured_output_by_default" + ), + } + model["model_info"] = ( + {**merged_model_info, "supports_native_structured_output": native_structured_output_support(native_output_info)} + if "supports_native_structured_output" in merged_model_info + and "supports_native_structured_output" not in stamped_model_info + else merged_model_info + ) # don't return the api key / vertex credentials # don't return the llm credentials model = remove_sensitive_info_from_deployment(model, excluded_keys={"litellm_credential_name"}) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5a14dc95e7c..b082198d3f2 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -217,6 +217,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_native_streaming: bool | None supports_native_structured_output: bool | None supports_bedrock_native_structured_output: ReadOnly[bool | None] + supports_bedrock_native_structured_output_by_default: ReadOnly[bool | None] supports_parallel_function_calling: bool | None supports_web_search: bool | None supports_reasoning: bool | None @@ -4303,6 +4304,7 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( "supports_regex_lookaround", "supports_assistant_prefill", "supports_bedrock_native_structured_output", + "supports_bedrock_native_structured_output_by_default", "supports_per_turn_control", "supports_pdf_input", "supports_continuation", diff --git a/litellm/utils.py b/litellm/utils.py index 8944ecb5648..13f22ddc36e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4167,6 +4167,13 @@ def _registration_base( return _RegistrationWrite(key=key, entry=dict(fallback.entry)) +def native_structured_output_support(model_info: Mapping[str, object]) -> bool | None: + if model_info.get("supports_bedrock_native_structured_output_by_default"): + return model_info.get("supports_bedrock_native_structured_output") is not False + declared: Final = model_info.get("supports_native_structured_output") + return declared if isinstance(declared, bool) else None + + def _base_fields_replaced_by(value: Mapping[str, object]) -> frozenset[str]: return ( frozenset({"explicit_cache_read_input_token_cost"}) @@ -7587,10 +7594,13 @@ def get_model_info_helper( supports_pdf_input=_model_info.get("supports_pdf_input", None), supports_embedding_image_input=_model_info.get("supports_embedding_image_input", None), supports_native_streaming=_model_info.get("supports_native_streaming", None), - supports_native_structured_output=_model_info.get("supports_native_structured_output", None), + supports_native_structured_output=native_structured_output_support(_model_info), supports_bedrock_native_structured_output=_model_info.get( "supports_bedrock_native_structured_output", None ), + supports_bedrock_native_structured_output_by_default=_model_info.get( + "supports_bedrock_native_structured_output_by_default", None + ), supports_web_search=_model_info.get("supports_web_search", None), supports_url_context=_model_info.get("supports_url_context", None), supports_continuation=_model_info.get("supports_continuation", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4e81afb0ea6..d9d996208d8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1961,6 +1961,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2041,6 +2042,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2116,6 +2118,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2242,7 +2245,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2287,6 +2291,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2331,7 +2336,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2376,6 +2382,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2420,7 +2427,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2465,6 +2473,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2504,7 +2513,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2549,6 +2559,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2593,7 +2604,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2633,6 +2645,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2715,7 +2728,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2759,7 +2773,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2803,7 +2818,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2847,7 +2863,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2891,7 +2908,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3023,6 +3041,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3068,6 +3087,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3113,6 +3133,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3158,6 +3179,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3203,6 +3225,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -75549,7 +75572,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -75589,7 +75613,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -75629,6 +75654,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -75702,6 +75728,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77417,6 +77444,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77474,6 +77502,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77651,6 +77680,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77703,6 +77733,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77750,6 +77781,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77863,6 +77895,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, diff --git a/tests/integration/providers/test_bedrock_batch_files_wire.py b/tests/integration/providers/test_bedrock_batch_files_wire.py index 823bcda7e06..1657895109d 100644 --- a/tests/integration/providers/test_bedrock_batch_files_wire.py +++ b/tests/integration/providers/test_bedrock_batch_files_wire.py @@ -88,10 +88,12 @@ _SCHEMA: Final = { @pytest.mark.parametrize( - "backend", ["us.anthropic.claude-haiku-5-5", "us.anthropic.claude-opus-4-8"], ids=["haiku_5_5", "opus_4_8"] + ("backend", "native_output"), + [("us.anthropic.claude-sonnet-5-5", False), ("us.anthropic.claude-haiku-5-5", True)], + ids=["opt_in_native_output_model", "native_output_model"], ) -def test_json_schema_batch_record_with_thinking_uploads_the_json_tool_and_no_native_output( - gateway: Gateway, backend: str +def test_json_schema_batch_record_with_thinking_uploads_native_output_only_on_models_that_always_take_it( + gateway: Gateway, backend: str, native_output: bool ) -> None: record: Final = { "custom_id": "schema-record", @@ -125,7 +127,13 @@ def test_json_schema_batch_record_with_thinking_uploads_the_json_tool_and_no_nat (upload,) = wire.drain() (stored,) = tuple(JSON_OBJECT.validate_json(line) for line in upload.body.decode().splitlines() if line.strip()) model_input: Final = object_value(stored["modelInput"]) - assert "format" not in object_value(model_input.get("output_config", {})), model_input - (tool,) = (object_value(entry) for entry in list_value(model_input["tools"])) + output_format: Final = object_value(model_input.get("output_config", {})).get("format") + tools: Final = tuple(object_value(entry) for entry in list_value(model_input.get("tools", []))) + if native_output: + assert object_value(object_value(output_format)["schema"])["properties"] == _SCHEMA["properties"], model_input + assert not tools, model_input + return + assert output_format is None, model_input + (tool,) = tools assert tool["name"] == _JSON_TOOL, model_input assert object_value(tool["input_schema"])["properties"] == _SCHEMA["properties"], model_input diff --git a/tests/integration/providers/test_bedrock_native_structured_output_wire.py b/tests/integration/providers/test_bedrock_native_structured_output_wire.py index 2502bd90bd6..68942ed295b 100644 --- a/tests/integration/providers/test_bedrock_native_structured_output_wire.py +++ b/tests/integration/providers/test_bedrock_native_structured_output_wire.py @@ -18,9 +18,10 @@ from pydantic import JsonValue, TypeAdapter _HAIKU: Final = "us.anthropic.claude-haiku-5-5" _FABLE: Final = "us.anthropic.claude-fable-5" _SONNET: Final = "us.anthropic.claude-sonnet-4-6" -_OPUS: Final = "us.anthropic.claude-opus-5-5" -_OPT_IN: Final = _OPUS -_UNFORCEABLE_TOOL: Final = _OPUS +_NEITHER_KEY: Final = "us.anthropic.claude-opus-4-7" +_OPT_IN: Final = _NEITHER_KEY +_BEDROCK_KEY_ONLY: Final = "us.anthropic.claude-sonnet-5-5" +_UNFORCEABLE_TOOL: Final = "anthropic.claude-opus-5-5" _OPUS_4_8: Final = "us.anthropic.claude-opus-4-8" _DEEPSEEK: Final = "deepseek.v3.2" _AWS: Final[dict[str, JsonValue]] = { @@ -683,7 +684,7 @@ def test_a_model_that_cannot_force_the_json_tool_retries_a_rejected_native_outpu @pytest.mark.parametrize( ("backend", "model_info", "native"), [ - (_OPUS, None, False), + (_NEITHER_KEY, None, False), (_OPT_IN, {"supports_bedrock_native_structured_output": True}, True), (_OPUS_4_8, {"supports_bedrock_native_structured_output": False}, False), ], @@ -897,7 +898,7 @@ def test_messages_on_bedrock_invoke_follows_the_bedrock_native_key_without_think ) -> None: marker, marker_hex = _marker() with wire_server(_bedrock) as wire, gateway.scenario() as scenario: - model: Final = _deployment(scenario, wire, "invoke", _HAIKU) + model: Final = _deployment(scenario, wire, "invoke", _BEDROCK_KEY_ONLY) response: Final = gateway.request( "POST", "/v1/messages", @@ -945,9 +946,9 @@ def test_malformed_schema_on_a_native_model_reaches_bedrock_once_without_a_proxy ("route", "backend", "thinking", "schema", "native"), [ ("converse", _SONNET, False, _NUMERIC_FORMATS_SCHEMA, True), - ("invoke", _HAIKU, True, _NUMERIC_FORMATS_SCHEMA, True), + ("invoke", _BEDROCK_KEY_ONLY, True, _NUMERIC_FORMATS_SCHEMA, True), ("converse", _SONNET, False, _WIDE_SCHEMA, True), - ("converse", _HAIKU, True, _WIDE_SCHEMA, False), + ("converse", _BEDROCK_KEY_ONLY, True, _WIDE_SCHEMA, False), ], ids=["numeric_formats_old_key", "numeric_formats_new_key_invoke", "wide_old_key", "wide_new_key"], ) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 146a6a3ff1c..0a9dc0ab3cf 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -496,7 +496,7 @@ def declare_mid_conversation_system(monkeypatch: pytest.MonkeyPatch) -> Callable return declare -NativeStructuredOutputKey: TypeAlias = Literal["legacy", "bedrock"] | None +NativeStructuredOutputKey: TypeAlias = Literal["legacy", "bedrock", "default"] | None @pytest.fixture @@ -506,7 +506,8 @@ def set_native_structured_output( def set_flags(model_cost_key: str, native_key: NativeStructuredOutputKey) -> None: entry: Final = litellm.model_cost[model_cost_key] monkeypatch.setitem(entry, "supports_native_structured_output", native_key == "legacy") - monkeypatch.setitem(entry, "supports_bedrock_native_structured_output", native_key == "bedrock") + monkeypatch.setitem(entry, "supports_bedrock_native_structured_output", native_key in ("bedrock", "default")) + monkeypatch.setitem(entry, "supports_bedrock_native_structured_output_by_default", native_key == "default") litellm.get_model_info.cache_clear() return set_flags diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py index 053574d7dd6..4f17f8b10e1 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py @@ -570,6 +570,7 @@ def _invoke_params_for_schema( ("us.anthropic.claude-opus-4-8", "bedrock", {"thinking": {"type": "adaptive"}}, True), ("us.anthropic.claude-opus-4-8", "bedrock", {"reasoning_effort": "low"}, True), ("us.anthropic.claude-haiku-5-5", "bedrock", {"thinking": {"type": "disabled"}}, False), + ("us.anthropic.claude-haiku-5-5", "default", {"thinking": {"type": "disabled"}}, True), ], ids=[ "legacy-key", @@ -578,6 +579,7 @@ def _invoke_params_for_schema( "bedrock-key-adaptive", "bedrock-key-effort", "bedrock-key-thinking-disabled", + "default-key-thinking-disabled", ], ) def test_bedrock_native_output_key_applies_only_when_the_model_will_think( @@ -2047,9 +2049,9 @@ def _invoke_chat_params( ) @pytest.mark.parametrize("model", ["us.anthropic.claude-opus-4-8", "global.anthropic.claude-sonnet-5"]) def test_invoke_uses_opt_in_native_output_when_the_opt_in_drove_native_output( - local_model_cost_map, model, thinking_params, monkeypatch + set_native_structured_output, model, thinking_params ): - monkeypatch.setitem(litellm.model_cost[model], "supports_native_structured_output", False) + set_native_structured_output(model, "bedrock") optional_params: Final = _invoke_chat_params( model, None, {"response_format": _CITY_RESPONSE_FORMAT, **thinking_params} ) @@ -2084,8 +2086,12 @@ def test_invoke_uses_opt_in_native_output_is_false_when_native_output_did_not_co ) -def test_invoke_uses_default_native_output_for_schema_retry(local_model_cost_map: None) -> None: - model: Final = "us.anthropic.claude-opus-5" +@pytest.mark.parametrize( + "model", + ["us.anthropic.claude-opus-5", "us.anthropic.claude-sonnet-5"], + ids=["opus-5", "sonnet-5"], +) +def test_invoke_uses_default_native_output_for_schema_retry(local_model_cost_map: None, model: str) -> None: optional_params: Final = _invoke_chat_params( model, None, {"response_format": _CITY_RESPONSE_FORMAT, "thinking": {"type": "disabled"}} ) diff --git a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py index 7d1b502df2b..d43cc771185 100644 --- a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py +++ b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py @@ -599,6 +599,139 @@ async def test_native_output_retry_is_bounded_by_the_request_timeout_not_the_str assert _sent_paths(respx_mock) == ["converse-stream", "converse"] +# Bedrock's 403 for a role without the streaming action, seen live on us.anthropic.claude-sonnet-5 on 2026-11-06 +_STREAM_PERMISSION_DENIAL: Final = json.dumps( + { + "Message": "User: arn:aws:sts::123456789012:assumed-role/invoke-only/qa is not authorized to perform: " + "bedrock:InvokeModelWithResponseStream on resource: arn:aws:bedrock:us-east-1:123456789012:inference-profile/" + "us.anthropic.claude-opus-5-5 because no identity-based policy allows the " + "bedrock:InvokeModelWithResponseStream action" + } +).encode() + + +def _denies_streaming_then_answers(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/converse-stream"): + return httpx.Response(403, content=_STREAM_PERMISSION_DENIAL, request=request) + return httpx.Response(200, content=CONVERSE_ANSWER, headers={"content-type": "application/json"}, request=request) + + +@pytest.mark.asyncio +async def test_a_denied_native_output_stream_is_sent_again_buffered_with_native_output( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post(url__regex=_BEDROCK_RUNTIME_URL).mock(side_effect=_denies_streaming_then_answers) + router: Final = _bedrock_router( + "bedrock/converse/us.anthropic.claude-opus-5-5", timeout=60, stream_timeout=STREAM_TIMEOUT_SECONDS + ) + + response: Final = await router.acompletion( + model="bedrock", + messages=[{"role": "user", "content": "hi"}], + stream=True, + thinking={"type": "disabled"}, + response_format={"type": "json_schema", "json_schema": {"name": "answer", "schema": _ANSWER_SCHEMA}}, + ) + + assert "".join([chunk.choices[0].delta.content or "" async for chunk in response]) == "hi" + assert _sent_paths(respx_mock) == ["converse-stream", "converse"] + retry_body: Final = json.loads(respx_mock.calls[1].request.content) + assert "textFormat" in retry_body.get("outputConfig", {}) and "toolConfig" not in retry_body + + +_INVOKE_ANSWER: Final = json.dumps( + { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-opus-5-5", + "content": [{"type": "text", "text": '{"answer": "hi"}'}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 5}, + } +).encode() + + +def _denies_invoke_streaming_then_answers(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/invoke-with-response-stream"): + return httpx.Response(403, content=_STREAM_PERMISSION_DENIAL, request=request) + return httpx.Response(200, content=_INVOKE_ANSWER, headers={"content-type": "application/json"}, request=request) + + +@pytest.mark.asyncio +async def test_a_denied_native_output_stream_on_the_invoke_route_is_sent_again_buffered_to_invoke( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post(url__regex=_BEDROCK_RUNTIME_URL).mock(side_effect=_denies_invoke_streaming_then_answers) + router: Final = _bedrock_router( + "bedrock/invoke/us.anthropic.claude-opus-5-5", timeout=60, stream_timeout=STREAM_TIMEOUT_SECONDS + ) + + response: Final = await router.acompletion( + model="bedrock", + messages=[{"role": "user", "content": "hi"}], + stream=True, + thinking={"type": "disabled"}, + response_format={"type": "json_schema", "json_schema": {"name": "answer", "schema": _ANSWER_SCHEMA}}, + ) + + assert "".join([chunk.choices[0].delta.content or "" async for chunk in response]) == '{"answer": "hi"}' + assert _sent_paths(respx_mock) == ["invoke-with-response-stream", "invoke"] + retry_body: Final = json.loads(respx_mock.calls[1].request.content) + assert "format" in retry_body.get("output_config", {}) and "tools" not in retry_body + + +_INVOKE_JSON_TOOL_ANSWER: Final = json.dumps( + { + "id": "msg_2", + "type": "message", + "role": "assistant", + "model": "claude-opus-5-5", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "json_tool_call", "input": {"answer": "hi"}}], + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 5}, + } +).encode() + + +def _denies_invoke_streaming_and_rejects_native_output(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/invoke-with-response-stream"): + return httpx.Response(403, content=_STREAM_PERMISSION_DENIAL, request=request) + if "format" in json.loads(request.content).get("output_config", {}): + return httpx.Response(400, content=_NATIVE_OUTPUT_REJECTION, request=request) + return httpx.Response( + 200, content=_INVOKE_JSON_TOOL_ANSWER, headers={"content-type": "application/json"}, request=request + ) + + +@pytest.mark.asyncio +async def test_a_rejected_schema_after_a_denied_invoke_stream_falls_back_to_the_json_tool_still_buffered( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post(url__regex=_BEDROCK_RUNTIME_URL).mock(side_effect=_denies_invoke_streaming_and_rejects_native_output) + router: Final = _bedrock_router( + "bedrock/invoke/us.anthropic.claude-opus-5-5", timeout=60, stream_timeout=STREAM_TIMEOUT_SECONDS + ) + + response: Final = await router.acompletion( + model="bedrock", + messages=[{"role": "user", "content": "hi"}], + stream=True, + thinking={"type": "disabled"}, + response_format={"type": "json_schema", "json_schema": {"name": "answer", "schema": _ANSWER_SCHEMA}}, + ) + + assert "".join([chunk.choices[0].delta.content or "" async for chunk in response]) == '{"answer": "hi"}' + assert _sent_paths(respx_mock) == ["invoke-with-response-stream", "invoke", "invoke"] + json_tool_body: Final = json.loads(respx_mock.calls[2].request.content) + assert "format" not in json_tool_body.get("output_config", {}) and "tools" in json_tool_body + + @pytest.mark.asyncio async def test_failed_native_output_retry_reports_the_request_timeout_it_was_bounded_by( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index b37d6977c10..4dcf312737d 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -5225,9 +5225,9 @@ def test_a_model_that_cannot_force_the_json_tool_gets_native_output_without_thin ("model", "thinking_params", "native"), [ ("us.anthropic.claude-opus-5-5", None, True), - ("us.anthropic.claude-opus-5-5", {"thinking": {"type": "adaptive"}}, False), + ("us.anthropic.claude-opus-5-5", {"thinking": {"type": "adaptive"}}, True), ("global.anthropic.claude-fable-5-1", None, True), - ("global.anthropic.claude-fable-5-1", {"reasoning_effort": "low"}, False), + ("global.anthropic.claude-fable-5-1", {"reasoning_effort": "low"}, True), ("us-gov.anthropic.claude-opus-5-5", None, False), ("us-gov.anthropic.claude-fable-5-1", None, False), ("us-gov-west-1/anthropic.claude-opus-5-5", None, False), @@ -5235,7 +5235,7 @@ def test_a_model_that_cannot_force_the_json_tool_gets_native_output_without_thin ("us-east-1/anthropic.claude-opus-5-5", None, True), ], ) -def test_models_whose_thinking_stops_under_native_output_keep_the_json_tool_only_when_thinking( +def test_native_output_follows_model_cost_map_for_bedrock_profiles( model: str, thinking_params: Mapping[str, object] | None, native: bool ) -> None: _, request = _converse_request_for_json_schema(model, False, thinking_params) @@ -5248,6 +5248,17 @@ def test_models_whose_thinking_stops_under_native_output_keep_the_json_tool_only @pytest.mark.parametrize( ("model", "thinking_params", "native"), [ + ("us.anthropic.claude-opus-5-5", {"thinking": {"type": "disabled"}}, True), + ("global.anthropic.claude-fable-5-1", {"thinking": {"type": "adaptive"}}, True), + ("global.anthropic.claude-fable-5-1", {"thinking": {"type": "disabled"}}, True), + ("us.anthropic.claude-sonnet-5", None, True), + ("us.anthropic.claude-sonnet-5", {"thinking": {"type": "adaptive"}}, True), + ("us.anthropic.claude-sonnet-5", {"thinking": {"type": "disabled"}}, True), + ("us.anthropic.claude-haiku-5-5", None, True), + ("us.anthropic.claude-haiku-5-5", {"thinking": {"type": "adaptive"}}, True), + ("us.anthropic.claude-haiku-5-5", {"thinking": {"type": "disabled"}}, True), + ("us-gov.anthropic.claude-sonnet-5", None, False), + ("us-gov.anthropic.claude-haiku-5-5", None, False), ("us.anthropic.claude-opus-5", None, True), ("us.anthropic.claude-opus-5", {"thinking": {"type": "adaptive"}}, True), ("us.anthropic.claude-opus-5", {"thinking": {"type": "disabled"}}, True), @@ -5258,6 +5269,17 @@ def test_models_whose_thinking_stops_under_native_output_keep_the_json_tool_only ("us-gov.anthropic.claude-opus-4-8", None, False), ], ids=[ + "opus-5-5-disabled", + "fable-5-1-adaptive", + "fable-5-1-disabled", + "sonnet-5-unset", + "sonnet-5-adaptive", + "sonnet-5-disabled", + "haiku-5-5-unset", + "haiku-5-5-adaptive", + "haiku-5-5-disabled", + "sonnet-5-us-gov", + "haiku-5-5-us-gov", "opus-5-unset", "opus-5-adaptive", "opus-5-disabled", @@ -5268,7 +5290,7 @@ def test_models_whose_thinking_stops_under_native_output_keep_the_json_tool_only "opus-4-8-us-gov", ], ) -def test_opus_5_and_4_8_cost_map_defaults_use_native_output_across_thinking_modes( +def test_model_cost_native_output_defaults_hold_across_thinking_modes( model: str, thinking_params: Mapping[str, object] | None, native: bool ) -> None: _, request = _converse_request_for_json_schema(model, False, thinking_params) @@ -5282,6 +5304,30 @@ def test_opus_5_and_4_8_cost_map_defaults_use_native_output_across_thinking_mode assert [tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]] == ["json_tool_call"] +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + ("model", "thinking_params"), + [ + ("us.anthropic.claude-opus-5-5", {"thinking": {"type": "adaptive"}}), + ("global.anthropic.claude-fable-5-1", {"thinking": {"type": "adaptive"}}), + ("us.anthropic.claude-sonnet-5", {"thinking": {"type": "disabled"}}), + ("us.anthropic.claude-haiku-5-5", None), + ("us.anthropic.claude-opus-5", None), + ], +) +@pytest.mark.parametrize("stream", [False, True]) +def test_deployment_opt_out_keeps_the_json_tool_on_models_that_default_to_native_output( + model: str, thinking_params: Mapping[str, object] | None, stream: bool +) -> None: + optional_params, request = _converse_request_for_json_schema( + model, stream, thinking_params, model_info={"supports_bedrock_native_structured_output": False} + ) + + assert "outputConfig" not in request + assert [tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]] == ["json_tool_call"] + assert optional_params.get("fake_stream", False) is stream + + def _converse_capital_answer(request: httpx.Request) -> httpx.Response: return httpx.Response( 200, @@ -5295,9 +5341,7 @@ def _converse_capital_answer(request: httpx.Request) -> httpx.Response: @pytest.mark.usefixtures("local_model_cost_map") -@pytest.mark.parametrize( - "model", ["us.anthropic.claude-opus-5-5", "global.anthropic.claude-sonnet-5-5", "global.anthropic.claude-fable-5-1"] -) +@pytest.mark.parametrize("model", ["global.anthropic.claude-sonnet-5-5"]) def test_streamed_json_schema_on_a_model_that_cannot_force_a_tool_still_calls_converse_without_streaming( model: str, ) -> None: @@ -5610,7 +5654,9 @@ def test_schema_past_a_native_output_size_limit_keeps_the_tool_path( @pytest.mark.parametrize( - ("native_key", "native_past_limit"), [("legacy", True), ("bedrock", False)], ids=["legacy-key", "bedrock-key"] + ("native_key", "native_past_limit"), + [("legacy", True), ("default", True), ("bedrock", False)], + ids=["legacy-key", "default-key", "bedrock-key"], ) def test_field_count_limit_sends_the_json_tool_only_on_the_bedrock_native_key( set_native_structured_output, native_key, native_past_limit @@ -10335,7 +10381,6 @@ def test_converse_reserves_the_tool_cache_point_within_the_cap(tool_cache_params assert request["toolConfig"]["tools"][-1] == cache_point - @pytest.mark.parametrize( "unsent_tool_params", [ @@ -10385,6 +10430,7 @@ def test_converse_leaves_the_cap_to_markers_when_a_marked_tool_is_not_sent( ] assert cache_point not in request.get("toolConfig", {}).get("tools", []) + def _converse_tool_turn(*result_parts: dict[str, object]) -> list[dict[str, object]]: return [ { @@ -10506,7 +10552,9 @@ def test_converse_translates_prompt_cache_breakpoint_on_an_unmapped_arn_only_whe messages=[ { "role": "user", - "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}}], + "content": [ + {"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}} + ], } ], optional_params={}, @@ -10518,7 +10566,6 @@ def test_converse_translates_prompt_cache_breakpoint_on_an_unmapped_arn_only_whe assert user_content(unflagged_arn, {"ttl": "30m"}) == [{"text": "Hello"}] - def test_converse_prompt_cache_breakpoint_follows_the_arn_flag_over_its_base_model(local_model_cost_map): from litellm import Router @@ -10545,7 +10592,9 @@ def test_converse_prompt_cache_breakpoint_follows_the_arn_flag_over_its_base_mod messages=[ { "role": "user", - "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}}], + "content": [ + {"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}} + ], } ], optional_params={}, @@ -10633,7 +10682,9 @@ def test_converse_prompt_cache_breakpoint_on_guarded_user_text_keeps_its_cache_p def test_converse_guarded_user_text_keeps_no_cache_point_on_a_model_not_known_to_take_one(local_model_cost_map): request: Final = litellm.AmazonConverseConfig()._transform_request( model="arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/unmapped1", - messages=[{"role": "user", "content": [{"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}]}], + messages=[ + {"role": "user", "content": [{"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}]} + ], optional_params={"guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "DRAFT"}}, litellm_params={}, ) @@ -11638,9 +11689,9 @@ _OUTPUT_CONFIG_FORMAT: Final = { ) @pytest.mark.parametrize("model", ["us.anthropic.claude-opus-4-8", "global.anthropic.claude-sonnet-5"]) def test_uses_opt_in_native_output_when_the_opt_in_drove_native_output( - local_model_cost_map, monkeypatch, model, request_params + set_native_structured_output, model, request_params ): - monkeypatch.setitem(litellm.model_cost[model], "supports_native_structured_output", False) + set_native_structured_output(model, "bedrock") optional_params: Final = _converse_params(model, None, request_params) assert AmazonConverseConfig().uses_opt_in_native_output(model, optional_params, {"model_info": {"id": "d1"}}) @@ -11672,9 +11723,9 @@ def test_uses_opt_in_native_output_is_false_when_native_output_did_not_come_from def test_uses_opt_in_native_output_reads_the_base_model_of_an_application_inference_profile( - local_model_cost_map, monkeypatch + set_native_structured_output, ): - monkeypatch.setitem(litellm.model_cost["us.anthropic.claude-opus-4-8"], "supports_native_structured_output", False) + set_native_structured_output("us.anthropic.claude-opus-4-8", "bedrock") profile: Final = "arn:aws:bedrock:us-east-1:000000000000:application-inference-profile/profile-id" litellm_params: Final = {"base_model": "bedrock/us.anthropic.claude-opus-4-8"} optional_params: Final = _converse_params( @@ -11768,8 +11819,9 @@ def test_a_refused_native_schema_is_sent_again_only_when_the_retry_drops_native_ @pytest.mark.respx(assert_all_called=True) +@pytest.mark.parametrize("model", ["us.anthropic.claude-sonnet-5", "us.anthropic.claude-opus-5"]) def test_a_default_native_schema_rejection_retries_with_the_json_tool( - respx_mock: respx.Router, local_model_cost_map: None + respx_mock: respx.Router, local_model_cost_map: None, model: str ) -> None: route: Final = respx_mock.post(url__regex=r"/converse$").mock( return_value=httpx.Response(400, json={"message": _BEDROCK_SCHEMA_REFUSAL}) @@ -11777,7 +11829,7 @@ def test_a_default_native_schema_rejection_retries_with_the_json_tool( with pytest.raises(litellm.BadRequestError, match=re.escape(_BEDROCK_SCHEMA_REFUSAL)): completion( - model="bedrock/converse/us.anthropic.claude-opus-5", + model=f"bedrock/converse/{model}", messages=[{"role": "user", "content": "What is the capital of France?"}], response_format=CAPITAL_RESPONSE_FORMAT, thinking={"type": "disabled"}, @@ -11938,7 +11990,9 @@ def test_converse_sends_no_tools_when_drop_params_drops_every_hosted_mcp_tool(mo def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: return tuple( - r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() ) @@ -12102,9 +12156,15 @@ def test_converse_sends_only_the_header_betas_when_header_and_body_both_carry_be def _five_cache_hint_conversation() -> list[dict[str, object]]: return [ - {"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]}, + { + "role": "system", + "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}], + }, {"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]}, - {"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]}, + { + "role": "assistant", + "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}], + }, {"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": "answer 2"}, {"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]}, @@ -12121,7 +12181,10 @@ def _five_cache_hint_conversation() -> list[dict[str, object]]: ], ) def test_converse_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params( - drop_params: bool | str, expected_system_cache_points: int, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None + drop_params: bool | str, + expected_system_cache_points: int, + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, ) -> None: monkeypatch.setattr(litellm, "drop_params", False) messages: Final = _five_cache_hint_conversation() diff --git a/tests/unit/llms/bedrock/chat/test_native_output_fallback.py b/tests/unit/llms/bedrock/chat/test_native_output_fallback.py index 81832d7d8bf..c87451d9577 100644 --- a/tests/unit/llms/bedrock/chat/test_native_output_fallback.py +++ b/tests/unit/llms/bedrock/chat/test_native_output_fallback.py @@ -149,6 +149,122 @@ def test_a_rejection_reaches_the_caller_when_the_retry_would_send_native_output_ assert bedrock.sent == (None,) +# Bedrock's 403 for a role without the streaming action, seen live on us.anthropic.claude-sonnet-5 on 2026-11-06 +_STREAM_PERMISSION_DENIAL: Final = BedrockError( + status_code=403, + message='{"Message":"User: arn:aws:sts::123456789012:assumed-role/invoke-only/qa is not authorized to perform: ' + "bedrock:InvokeModelWithResponseStream on resource: arn:aws:bedrock:us-west-2:123456789012:inference-profile/" + "us.anthropic.claude-sonnet-5 because no identity-based policy allows the " + 'bedrock:InvokeModelWithResponseStream action"}', +) +_INVOKE_PERMISSION_DENIAL: Final = BedrockError( + status_code=403, + message='{"Message":"User: arn:aws:sts::123456789012:assumed-role/no-bedrock/qa is not authorized to perform: ' + 'bedrock:InvokeModel because no identity-based policy allows the bedrock:InvokeModel action"}', +) +_BUFFERED_NATIVE_STREAM: Final = NativeOutputFallback( + optional_params={"outputConfig": "native", "fake_stream": True}, litellm_params={} +) + + +def _buffered(fallback: NativeOutputFallback) -> NativeOutputFallback: + return NativeOutputFallback( + optional_params={**fallback.optional_params, "fake_stream": True}, litellm_params=fallback.litellm_params + ) + + +def _call_with_buffered_stream_fallback( + bedrock: ScriptedBedrock, fallback: NativeOutputFallback | None, buffered_stream_fallback: NativeOutputFallback +) -> object: + return _resolve( + call_with_native_output_fallback( + bedrock, + None if fallback is None else lambda: fallback, + _MODEL, + buffered_stream_fallback=lambda: buffered_stream_fallback, + ) + ) + + +@_MODES +def test_a_denied_native_stream_is_sent_again_once_as_a_buffered_native_request( + asynchronous: bool, caplog: pytest.LogCaptureFixture +) -> None: + bedrock: Final = ScriptedBedrock(errors=(_STREAM_PERMISSION_DENIAL, None), asynchronous=asynchronous) + + with caplog.at_level("WARNING", logger="LiteLLM"): + response: Final = _call_with_buffered_stream_fallback(bedrock, _FALLBACK, _BUFFERED_NATIVE_STREAM) + + assert isinstance(response, ModelResponse) and response.id == "answer-2" + assert bedrock.sent == (None, _BUFFERED_NATIVE_STREAM) + assert "bedrock:InvokeModelWithResponseStream" in caplog.text and _MODEL in caplog.text + + +@_MODES +def test_a_buffered_native_request_bedrock_rejects_is_sent_again_buffered_with_the_fallback_params( + asynchronous: bool, +) -> None: + bedrock: Final = ScriptedBedrock( + errors=(_STREAM_PERMISSION_DENIAL, _GRAMMAR_REJECTION, None), asynchronous=asynchronous + ) + + response: Final = _call_with_buffered_stream_fallback(bedrock, _FALLBACK, _BUFFERED_NATIVE_STREAM) + + assert isinstance(response, ModelResponse) and response.id == "answer-3" + assert bedrock.sent == (None, _BUFFERED_NATIVE_STREAM, _buffered(_FALLBACK)) + + +@_MODES +def test_strict_tools_bedrock_rejects_after_a_buffered_native_retry_are_dropped_on_the_buffered_json_tool_retry( + asynchronous: bool, +) -> None: + bedrock: Final = ScriptedBedrock( + errors=(_STREAM_PERMISSION_DENIAL, _GRAMMAR_REJECTION, _GRAMMAR_REJECTION, None), asynchronous=asynchronous + ) + + response: Final = _resolve( + call_with_native_output_fallback( + bedrock, + lambda: _NATIVE_RETRY_WITH_STRICT_TOOLS, + "us.anthropic.claude-sonnet-4-6", + strict_tools_fallback=lambda: _STRICT_FALLBACK, + buffered_stream_fallback=lambda: _BUFFERED_NATIVE_STREAM, + ) + ) + + assert isinstance(response, ModelResponse) and response.id == "answer-4" + assert bedrock.sent == ( + None, + _BUFFERED_NATIVE_STREAM, + _buffered(_NATIVE_RETRY_WITH_STRICT_TOOLS), + _buffered(_COMPOSED_FALLBACK), + ) + + +@_MODES +@pytest.mark.parametrize( + ("error", "buffered_stream_fallback"), + [ + ( + _STREAM_PERMISSION_DENIAL, + NativeOutputFallback(optional_params={"outputConfig": "native"}, litellm_params={}), + ), + (_INVOKE_PERMISSION_DENIAL, _BUFFERED_NATIVE_STREAM), + ], + ids=["retry_would_stream_again", "invoke_denied"], +) +def test_a_permission_denial_reaches_the_caller_unless_the_retry_buffers_the_stream( + asynchronous: bool, error: BedrockError, buffered_stream_fallback: NativeOutputFallback +) -> None: + bedrock: Final = ScriptedBedrock(errors=(error,), asynchronous=asynchronous) + + with pytest.raises(BedrockError) as raised: + _call_with_buffered_stream_fallback(bedrock, _FALLBACK, buffered_stream_fallback) + + assert raised.value is error + assert bedrock.sent == (None,) + + @pytest.mark.parametrize( ("error", "advises_opting_out"), [(_REJECTION, True), (_SCHEMA_REFUSAL, False)], diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index 93bda5b444d..fdd6d5e61bc 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -21,7 +21,7 @@ from botocore.credentials import Credentials import litellm from litellm.constants import DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET -from litellm.llms.bedrock.common_utils import BedrockError +from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES, BedrockError from litellm.llms.bedrock.files.transformation import BedrockFilesConfig, BedrockJsonlFilesTransformation from tests.unit.proxy.conftest import httpx_transport @@ -2069,6 +2069,76 @@ class TestBedrockBatchAnthropicRowParams: assert ("format" in model_input.get("output_config", {})) is native_output assert (tool_names == {"json_tool_call"}) is not native_output + @pytest.mark.parametrize( + ("model", "native_over_the_limit"), + [(MODEL, True), ("bedrock/us.anthropic.claude-opus-5-5", False)], + ids=["already_native_model", "native_by_default_model"], + ) + @pytest.mark.parametrize("over_the_limit", [False, True], ids=["at_the_limit", "over_the_limit"]) + def test_only_native_by_default_models_send_json_schemas_wider_than_the_limit_through_the_json_tool( + self, local_model_cost_map, model, native_over_the_limit, over_the_limit + ): + native_output: Final = native_over_the_limit or not over_the_limit + field_names: Final = tuple( + f"field_{index:03d}" for index in range(BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES + over_the_limit) + ) + wide_schema: Final = { + "type": "object", + "properties": {name: {"type": "string"} for name in field_names}, + "required": list(field_names), + "additionalProperties": False, + } + model_input: Final = self._transform( + "/v1/chat/completions", + { + "messages": [{"role": "user", "content": "Describe a ceramic coffee mug"}], + "response_format": {"type": "json_schema", "json_schema": {"name": "wide", "schema": wide_schema}}, + }, + model=model, + ) + tool_names: Final = frozenset(tool["name"] for tool in model_input.get("tools", [])) + + assert ("format" in model_input.get("output_config", {})) is native_output + assert (tool_names == {"json_tool_call"}) is not native_output + + def test_a_deployment_opt_out_sends_uploaded_json_schema_rows_through_the_json_tool(self, local_model_cost_map): + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + record: Final = { + "custom_id": "row-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": self.MODEL, + "messages": [{"role": "user", "content": "Weather in Paris?"}], + "response_format": { + "type": "json_schema", + "json_schema": {"name": "weather", "schema": self.PARAMETERS}, + }, + }, + } + request: Final = BedrockFilesConfig().transform_create_file_request( + model=self.MODEL, + create_file_data={ + "file": ("batch.jsonl", json.dumps(record).encode(), "application/jsonl"), + "purpose": "batch", + }, + optional_params={ + "aws_access_key_id": "test-key-id", + "aws_secret_access_key": "test-secret", + "aws_region_name": "us-west-2", + }, + litellm_params={ + "s3_bucket_name": "litellm-batch-bucket", + "litellm_metadata": {"model_info": {"supports_bedrock_native_structured_output": False}}, + }, + ) + assert isinstance(request, dict) + model_input: Final = json.loads(request["data"].splitlines()[0])["modelInput"] + + assert "format" not in model_input.get("output_config", {}) + assert [tool["name"] for tool in model_input["tools"]] == ["json_tool_call"] + def test_provider_native_params_still_pass_through(self): model_input = self._transform( "/v1/chat/completions", diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index 0c1c4f5475a..7cc4629e497 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -1,11 +1,10 @@ - import asyncio, importlib, litellm, pytest import respx from httpx import Response from typing import Final -from litellm.llms.bedrock.common_utils import( +from litellm.llms.bedrock.common_utils import ( AmazonBedrockGlobalConfig, BedrockModelInfo, NativeOutputFallback, @@ -54,9 +53,7 @@ def test_bedrock_response_stream_shape_lazy_loads_once(): import litellm.llms.bedrock.common_utils as mod sentinel = MagicMock() - with patch.object( - mod, "_load_bedrock_response_stream_shape", return_value=sentinel - ) as mock_load: + with patch.object(mod, "_load_bedrock_response_stream_shape", return_value=sentinel) as mock_load: assert mod.get_bedrock_response_stream_shape() is sentinel assert mod.get_bedrock_response_stream_shape() is sentinel mock_load.assert_called_once() @@ -103,9 +100,7 @@ def test_bedrock_response_stream_shape_is_structure_shape(): from litellm.llms.bedrock.common_utils import get_bedrock_response_stream_shape loaded_shape = get_bedrock_response_stream_shape() - assert ( - loaded_shape is not None - ), "get_bedrock_response_stream_shape() is None — botocore may not be installed" + assert loaded_shape is not None, "get_bedrock_response_stream_shape() is None — botocore may not be installed" shape: StructureShape = loaded_shape assert isinstance(shape, StructureShape) assert shape.name == "ResponseStream" @@ -170,9 +165,7 @@ def test_deepseek_cris(): Test that DeepSeek models with cross-region inference prefix use converse route """ bedrock_model_info = BedrockModelInfo - bedrock_route = bedrock_model_info.get_bedrock_route( - model="bedrock/us.deepseek.r1-v1:0" - ) + bedrock_route = bedrock_model_info.get_bedrock_route(model="bedrock/us.deepseek.r1-v1:0") assert bedrock_route == "converse" @@ -228,7 +221,10 @@ def test_bare_opaque_arn_with_no_invoke_provider_routes_to_converse(arn_segment: model: Final = f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}" assert BaseAWSLLM.get_bedrock_invoke_provider(model) is None assert BedrockModelInfo.get_bedrock_route(model) == "converse" - assert BedrockModelInfo.get_bedrock_route(f"bedrock/llama/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}") == "invoke" + assert ( + BedrockModelInfo.get_bedrock_route(f"bedrock/llama/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}") + == "invoke" + ) def test_govcloud_cross_region_inference_prefix(): @@ -238,27 +234,19 @@ def test_govcloud_cross_region_inference_prefix(): bedrock_model_info = BedrockModelInfo # Test us-gov prefix is stripped correctly for Claude models - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.anthropic.claude-haiku-4-5-20251001-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-haiku-4-5-20251001-v1:0") assert base_model == "anthropic.claude-haiku-4-5-20251001-v1:0" # Test us-gov prefix is stripped correctly for different Claude versions - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0") assert base_model == "anthropic.claude-sonnet-4-5-20250929-v1:0" # Test us-gov prefix is stripped correctly for Haiku models - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.anthropic.claude-3-haiku-20240307-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-3-haiku-20240307-v1:0") assert base_model == "anthropic.claude-3-haiku-20240307-v1:0" # Test us-gov prefix is stripped correctly for Meta models - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.meta.llama3-8b-instruct-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.meta.llama3-8b-instruct-v1:0") assert base_model == "meta.llama3-8b-instruct-v1:0" @@ -272,23 +260,14 @@ def test_context_window_suffix_stripped_for_cost_lookup(): """ from litellm.llms.bedrock.common_utils import get_bedrock_base_model - assert ( - get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1[1m]") - == "anthropic.claude-opus-4-6-v1" - ) - assert ( - get_bedrock_base_model("us.anthropic.claude-sonnet-4-6[1m]") - == "anthropic.claude-sonnet-4-6" - ) + assert get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1[1m]") == "anthropic.claude-opus-4-6-v1" + assert get_bedrock_base_model("us.anthropic.claude-sonnet-4-6[1m]") == "anthropic.claude-sonnet-4-6" assert ( get_bedrock_base_model("global.anthropic.claude-opus-4-5-20251101-v1:0[1m]") == "anthropic.claude-opus-4-5-20251101-v1:0" ) # Ensure models without suffix are unaffected - assert ( - get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1") - == "anthropic.claude-opus-4-6-v1" - ) + assert get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1") == "anthropic.claude-opus-4-6-v1" # Ensure :51k throughput suffix still works assert ( get_bedrock_base_model("anthropic.claude-3-5-sonnet-20241022-v2:0:51k") @@ -358,9 +337,7 @@ def test_output_config_effort_normalization_uses_model_info_ceiling(monkeypatch) ("us.anthropic.claude-opus-4-7", "xhigh"), ], ) -def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling( - model, expected_ceiling -): +def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling(model, expected_ceiling): from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap model_info = GetModelCostMap.load_local_model_cost_map()[model] @@ -379,54 +356,24 @@ def test_route_prefix_matched_as_path_segment_not_substring(): or a ``/`` boundary. """ # The bedrock_mantle/ provider prefix must NOT be read as the mantle/ route. - assert ( - BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.5") != "mantle" - ) - assert ( - BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.4") == "converse" - ) - assert ( - BedrockModelInfo._explicit_mantle_route("bedrock_mantle/openai.gpt-5.5") - is False - ) + assert BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.5") != "mantle" + assert BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.4") == "converse" + assert BedrockModelInfo._explicit_mantle_route("bedrock_mantle/openai.gpt-5.5") is False # A genuine mantle route still resolves, via the startswith branch... - assert ( - BedrockModelInfo.get_bedrock_route("mantle/anthropic.claude-mythos-preview") - == "mantle" - ) + assert BedrockModelInfo.get_bedrock_route("mantle/anthropic.claude-mythos-preview") == "mantle" # ...and via the mid-path "/mantle/" branch (after the bedrock/ provider prefix). - assert ( - BedrockModelInfo.get_bedrock_route( - "bedrock/mantle/anthropic.claude-mythos-preview" - ) - == "mantle" - ) + assert BedrockModelInfo.get_bedrock_route("bedrock/mantle/anthropic.claude-mythos-preview") == "mantle" def test_model_has_route_prefix_exercises_both_branches(): """``_model_has_route_prefix`` matches on ``startswith`` or a ``/`` boundary only.""" # startswith branch - assert ( - BedrockModelInfo._model_has_route_prefix( - "mantle/anthropic.claude-mythos-preview", "mantle/" - ) - is True - ) + assert BedrockModelInfo._model_has_route_prefix("mantle/anthropic.claude-mythos-preview", "mantle/") is True # f"/{prefix}" boundary branch - assert ( - BedrockModelInfo._model_has_route_prefix( - "bedrock/mantle/anthropic.claude-mythos-preview", "mantle/" - ) - is True - ) + assert BedrockModelInfo._model_has_route_prefix("bedrock/mantle/anthropic.claude-mythos-preview", "mantle/") is True # neither branch: the token only appears glued to another segment - assert ( - BedrockModelInfo._model_has_route_prefix( - "bedrock_mantle/openai.gpt-5.5", "mantle/" - ) - is False - ) + assert BedrockModelInfo._model_has_route_prefix("bedrock_mantle/openai.gpt-5.5", "mantle/") is False @pytest.mark.parametrize( @@ -476,16 +423,10 @@ def test_explicit_invoke_route_does_not_match_async_invoke(): """ async_invoke_model = "async_invoke/twelvelabs.marengo-embed-2-7-v1:0" assert BedrockModelInfo._explicit_invoke_route(async_invoke_model) is False - assert ( - BedrockModelInfo._explicit_invoke_route(f"bedrock/{async_invoke_model}") - is False - ) + assert BedrockModelInfo._explicit_invoke_route(f"bedrock/{async_invoke_model}") is False # ...while async_invoke/ is still detected as its own route. assert BedrockModelInfo._explicit_async_invoke_route(async_invoke_model) is True - assert ( - BedrockModelInfo._explicit_async_invoke_route(f"bedrock/{async_invoke_model}") - is True - ) + assert BedrockModelInfo._explicit_async_invoke_route(f"bedrock/{async_invoke_model}") is True def test_capability_lookups_fall_back_to_base_model_when_regional_entry_lacks_field(monkeypatch): @@ -605,7 +546,6 @@ def test_deployment_model_info_breakpoint_flag_covers_an_unmapped_arn(local_mode assert bedrock_model_accepts_cache_points(unflagged_arn) is True - _CONVERSE_REPLY: Final = { "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, "stopReason": "end_turn", @@ -638,7 +578,11 @@ _MARKED_TOOL_CONVERSATION: Final = ( } ], }, - {"role": "tool", "tool_call_id": "call_1", "content": [{"type": "text", "text": "Sunny", "cache_control": _CACHE_MARK}]}, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [{"type": "text", "text": "Sunny", "cache_control": _CACHE_MARK}], + }, {"role": "tool", "tool_call_id": "call_1_1", "content": "Cloudy"}, {"role": "assistant", "content": "Paris is sunny, Rome is cloudy.", "cache_control": _CACHE_MARK}, {"role": "user", "content": "Thanks"}, @@ -711,6 +655,7 @@ async def test_a_bedrock_deployment_sends_cache_points_by_its_own_flag_not_a_sib "flagless": not declared, } + async def test_a_routed_bedrock_deployment_sends_cache_points_when_register_model_opted_its_model_in( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None ) -> None: @@ -738,7 +683,9 @@ async def test_a_routed_bedrock_deployment_sends_cache_points_when_register_mode num_retries=0, ) - await router.acompletion(model="flagless", messages=[{"role": "system", "content": "stable"}, {"role": "user", "content": "hi"}]) + await router.acompletion( + model="flagless", messages=[{"role": "system", "content": "stable"}, {"role": "user", "content": "hi"}] + ) assert b"cachePoint" in converse.calls.last.request.content @@ -783,10 +730,13 @@ def test_a_deployments_prompt_caching_flag_written_as_a_string_or_number_decides ) assert ( - bedrock_model_accepts_cache_points(model, model_info={"id": "qa-deployment", "supports_prompt_caching": declared}) + bedrock_model_accepts_cache_points( + model, model_info={"id": "qa-deployment", "supports_prompt_caching": declared} + ) is expected ) + def test_merge_bedrock_aws_request_params_strips_caller_identity_when_deployment_has_static_credentials(): from litellm.llms.bedrock.common_utils import merge_bedrock_aws_request_params @@ -1491,6 +1441,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="session") def event_loop(): try: @@ -1500,6 +1451,7 @@ def event_loop(): yield loop loop.close() + @pytest.fixture(scope="function") def setup_and_teardown(event_loop): import litellm @@ -1533,6 +1485,7 @@ def setup_and_teardown(event_loop): if pending: event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "set_verbose": getattr(litellm, "set_verbose", False), @@ -1547,6 +1500,7 @@ _SCALAR_DEFAULTS = { "cohere_key": getattr(litellm, "cohere_key", None), } + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestStripBedrockRoutingPrefix: """Tests for strip_bedrock_routing_prefix function.""" @@ -1577,6 +1531,7 @@ class TestStripBedrockRoutingPrefix: == "anthropic.claude-3-sonnet-20240229-v1:0" ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestStripBedrockThroughputSuffix: """Tests for strip_bedrock_throughput_suffix function.""" @@ -1604,6 +1559,7 @@ class TestStripBedrockThroughputSuffix: def test_strip_throughput_suffix(self, input_model, expected): assert strip_bedrock_throughput_suffix(input_model) == expected + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestExtractModelNameFromBedrockArn: """Tests for extract_model_name_from_bedrock_arn function.""" @@ -1624,6 +1580,7 @@ class TestExtractModelNameFromBedrockArn: arn = "ARN:aws:bedrock:us-east-1:123456789012:model/my-model" assert extract_model_name_from_bedrock_arn(arn) == "my-model" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestGetBedrockCrossRegionInferenceRegions: """Tests for get_bedrock_cross_region_inference_regions function.""" @@ -1639,6 +1596,7 @@ class TestGetBedrockCrossRegionInferenceRegions: regions = get_bedrock_cross_region_inference_regions() assert isinstance(regions, list) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestGetBedrockBaseModel: """Tests for get_bedrock_base_model function.""" @@ -1702,6 +1660,7 @@ class TestGetBedrockBaseModel: """Test that throughput tier suffixes like :51k are stripped. Issue #19113.""" assert get_bedrock_base_model(input_model) == expected + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestBedrockModelInfoWrappers: """Tests that BedrockModelInfo methods correctly wrap standalone functions.""" @@ -1723,6 +1682,7 @@ class TestBedrockModelInfoWrappers: model = "bedrock/converse/claude-3" assert BedrockModelInfo.get_non_litellm_routing_model_name(model) == strip_bedrock_routing_prefix(model) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestBedrockTokenCounter: """Tests for BedrockTokenCounter class.""" @@ -1759,12 +1719,14 @@ class TestBedrockTokenCounter: ) assert result is None + @pytest.fixture def _pr4_bedrock_env(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("AWS_ACCESS_KEY_ID", "pr4-test-aws-access-key") monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "pr4-test-aws-secret-key") monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + @pytest.fixture(scope="module") def _use_local_model_cost_map() -> Iterator[None]: with pytest.MonkeyPatch.context() as monkeypatch: @@ -1772,6 +1734,7 @@ def _use_local_model_cost_map() -> Iterator[None]: importlib.reload(litellm) yield + @pytest.mark.usefixtures( "_pr4_bedrock_env", "_use_local_model_cost_map", @@ -2348,8 +2311,12 @@ def test_bedrock_capability_model_reads_base_model_only_for_application_profile_ assert bedrock_capability_model(model, base_model) == expected -_FORMAT_REJECTION: Final = "The model returned the following errors: output_config.format: Extra inputs are not permitted" -_COMPILE_REJECTION: Final = "The model returned the following errors: compilation exceeded limits: Schema is too complex." +_FORMAT_REJECTION: Final = ( + "The model returned the following errors: output_config.format: Extra inputs are not permitted" +) +_COMPILE_REJECTION: Final = ( + "The model returned the following errors: compilation exceeded limits: Schema is too complex." +) _GRAMMAR_REJECTION: Final = ( "The model returned the following errors: The compiled grammar is too large, which would cause performance issues. " "Simplify your tool schemas or reduce the number of strict tools." @@ -2394,10 +2361,10 @@ def test_is_bedrock_native_output_rejection_matches_only_the_400s_that_reject_na ("model", "model_info", "expected"), [ ("us.anthropic.claude-opus-4-8", None, False), - ("global.anthropic.claude-sonnet-5", None, True), + ("global.anthropic.claude-sonnet-5", None, False), ("us-gov.anthropic.claude-opus-4-8", None, False), ("us.anthropic.claude-opus-4-8", {"supports_bedrock_native_structured_output": False}, False), - ("us.anthropic.claude-opus-5-5", {"supports_bedrock_native_structured_output": True}, True), + ("us.anthropic.claude-opus-5-5", {"supports_bedrock_native_structured_output": True}, False), ("us.anthropic.claude-sonnet-4-6", {"supports_bedrock_native_structured_output": True}, False), ], ids=["opus_4_8", "sonnet_5", "govcloud_opus_4_8", "deployment_opt_out", "deployment_opt_in", "native_model"], @@ -2490,7 +2457,10 @@ def test_bedrock_strict_tools_fallback_drops_only_strict_and_keeps_everything_el assert fallback == NativeOutputFallback( optional_params={ "tools": [ - {**strict_tool, "function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}}}, + { + **strict_tool, + "function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}}, + }, plain_tool, anthropic_tool, {"toolSpec": {"name": "spec", "inputSchema": {"json": {"type": "object"}}}}, diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index dd0e5762845..a758f4d4250 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -4890,6 +4890,92 @@ async def test_json_mode_stream_reaches_the_iterator_of_a_config_without_a_reque ] +class _PlainStreamRouteConfig(_TransformRecordingConfig): + def __init__(self): + super().__init__(transform_async=False) + + def get_complete_url(self, api_base, api_key, model, optional_params, litellm_params, stream=None): + return f"{api_base}/{'stream' if stream else 'answer'}" + + +class _StreamRouteConfig(_PlainStreamRouteConfig): + @property + def has_custom_stream_wrapper(self) -> bool: + return True + + +def _answers_only_the_non_streaming_route(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/answer"): + return httpx.Response(200, json=json.loads(request.content)) + return httpx.Response(403, json={"message": "not authorized to stream"}) + + +def _fake_streamed_completion( + acompletion: bool, + client: HTTPHandler | AsyncHTTPHandler, + fake_stream: object = True, + provider_config: BaseConfig | None = None, +): + return BaseLLMHTTPHandler().completion( + model="stub-model", + messages=[{"role": "user", "content": "hi"}], + api_base="https://llm.example/v1/chat", + custom_llm_provider="openai", + model_response=ModelResponse(), + encoding=None, + logging_obj=_streaming_logging_obj(), + optional_params={"fake_stream": fake_stream}, + timeout=10.0, + litellm_params={}, + acompletion=acompletion, + stream=True, + client=client, + provider_config=provider_config or _StreamRouteConfig(), + ) + + +async def test_a_fake_streamed_completion_is_sent_to_the_non_streaming_route(): + sync_client: Final = HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_answers_only_the_non_streaming_route)) + ) + async_client: Final = AsyncHTTPHandler() + async_client.client = httpx.AsyncClient(transport=httpx.MockTransport(_answers_only_the_non_streaming_route)) + + sync_chunks: Final = list(_fake_streamed_completion(acompletion=False, client=sync_client)) + async_chunks: Final = [ + chunk async for chunk in await _fake_streamed_completion(acompletion=True, client=async_client) + ] + + assert (_streamed_text(sync_chunks), _streamed_text(async_chunks)) == ("sync", "sync") + + +def _answers_only_the_streaming_route(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/stream"): + return _sse_echoing_transformed_by(request) + return httpx.Response(403, json={"message": "only streaming is allowed"}) + + +@pytest.mark.parametrize("fake_stream", ["true", 1], ids=["string", "int"]) +async def test_a_fake_stream_value_other_than_true_still_streams_from_the_streaming_route(fake_stream: object): + sync_client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_answers_only_the_streaming_route))) + async_client: Final = AsyncHTTPHandler() + async_client.client = httpx.AsyncClient(transport=httpx.MockTransport(_answers_only_the_streaming_route)) + + sync_chunks: Final = list( + _fake_streamed_completion( + acompletion=False, client=sync_client, fake_stream=fake_stream, provider_config=_PlainStreamRouteConfig() + ) + ) + async_chunks: Final = [ + chunk + async for chunk in await _fake_streamed_completion( + acompletion=True, client=async_client, fake_stream=fake_stream, provider_config=_PlainStreamRouteConfig() + ) + ] + + assert (_streamed_text(sync_chunks), _streamed_text(async_chunks)) == ("sync", "sync") + + class _ImageEditRecordingConfig(BaseImageEditConfig): def __init__(self): self.transform_calls = [] diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 6ad5ef1a4eb..79f3e697a4f 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -490,6 +490,51 @@ def test_model_info_reports_null_cost_for_unpriced_deployment_and_zero_for_decla assert input_cost > 0 and output_cost > 0 +@pytest.mark.parametrize( + ("deployment_model_info", "native"), + [ + ({}, True), + ({"supports_bedrock_native_structured_output": False}, False), + ({"supports_native_structured_output": True, "supports_bedrock_native_structured_output": False}, True), + ], + ids=["native-by-default", "bedrock-opt-out", "explicit-generic-flag"], +) +def test_model_info_reports_a_deployment_bedrock_native_output_opt_out( + local_model_cost_map: None, deployment_model_info: dict[str, object], native: bool +) -> None: + info: Final = proxy_server._enrich_model_info_with_litellm_data( + { + "model_name": "sonnet", + "litellm_params": {"model": "bedrock/us.anthropic.claude-sonnet-5"}, + "model_info": {"id": "sonnet-deployment", **deployment_model_info}, + } + )["model_info"] + + assert info["supports_native_structured_output"] is native + + +@pytest.mark.parametrize( + ("model", "by_default"), + [("bedrock/us.anthropic.claude-opus-4-7", True), ("bedrock/us.anthropic.claude-sonnet-5", False)], + ids=["set-on-a-model-the-cost-map-leaves-on-the-json-tool", "unset-on-a-model-the-cost-map-makes-native"], +) +def test_model_info_ignores_a_deployment_by_default_key_the_router_reads_only_from_the_cost_map( + local_model_cost_map: None, model: str, by_default: bool +) -> None: + def reported_native(deployment_model_info: dict[str, object]) -> object: + return proxy_server._enrich_model_info_with_litellm_data( + { + "model_name": "claude", + "litellm_params": {"model": model}, + "model_info": {"id": "claude-deployment", **deployment_model_info}, + } + )["model_info"]["supports_native_structured_output"] + + assert reported_native({"supports_bedrock_native_structured_output_by_default": by_default}) is reported_native( + {} + ) + + def test_model_info_id_lookup_reports_the_same_cost_as_the_list( client: TestClient, auth_as: Callable[[], AbstractContextManager[object]], diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index c0c8a0655fe..cd71c3f9bee 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -542,7 +542,13 @@ def test_should_not_pollute_shared_key_with_custom_nonzero_pricing(): @pytest.mark.parametrize( "flag", - ["supports_regex_lookaround", "supports_assistant_prefill", "supports_sampling_params", "supports_continuation"], + [ + "supports_regex_lookaround", + "supports_assistant_prefill", + "supports_sampling_params", + "supports_continuation", + "supports_bedrock_native_structured_output_by_default", + ], ) def test_capability_flag_stays_on_the_deployment_that_set_it(flag: str) -> None: """A deployment's capability override must not land on the shared diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index f8ec0e56e4a..b0c55d3f0f2 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1354,6 +1354,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "supports_image_size": {"type": "boolean"}, "supports_native_structured_output": {"type": "boolean"}, "supports_bedrock_native_structured_output": {"type": "boolean"}, + "supports_bedrock_native_structured_output_by_default": {"type": "boolean"}, "use_openai_responses_path": {"type": "boolean"}, "off_peak_pricing": { "type": "object", diff --git a/tests/unit/test_utils_get_model_info.py b/tests/unit/test_utils_get_model_info.py index d08742be4af..7548001ce4c 100644 --- a/tests/unit/test_utils_get_model_info.py +++ b/tests/unit/test_utils_get_model_info.py @@ -13,7 +13,7 @@ from litellm import get_model_info from litellm.llms.bedrock.common_utils import BedrockModelInfo from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.types.utils import ModelInfoBase -from litellm.utils import _invalidate_model_cost_lowercase_map +from litellm.utils import _invalidate_model_cost_lowercase_map, supports_native_structured_output import os from unittest.mock import MagicMock, patch @@ -388,6 +388,29 @@ def test_get_model_info_bedrock_cross_region_capability_parity(): assert checked > 0, "no cross-region bedrock profiles found - the filter is inert" +def test_bedrock_native_output_default_reaches_only_versions_that_read_its_key(local_model_cost_map): + flagged: Final = tuple( + key for key, row in litellm.model_cost.items() if row.get("supports_bedrock_native_structured_output_by_default") + ) + + assert flagged + assert [key for key in flagged if litellm.model_cost[key].get("supports_native_structured_output")] == [] + assert [key for key in flagged if not get_model_info(key)["supports_native_structured_output"]] == [] + + +@pytest.mark.parametrize( + ("registered", "native"), + [({"supports_bedrock_native_structured_output": False}, False), ({"max_output_tokens": 1024}, True)], + ids=["bedrock-opt-out", "unrelated-override"], +) +def test_a_registered_bedrock_opt_out_beats_the_native_output_default(local_model_cost_map, registered, native): + model: Final = "us.anthropic.claude-sonnet-5" + litellm.register_model({model: registered}, persist_across_reloads=False) + + assert get_model_info(f"bedrock/{model}")["supports_native_structured_output"] is native + assert supports_native_structured_output(f"bedrock/{model}") is native + + def _is_positive_cost(value: object) -> bool: return isinstance(value, (int, float)) and value > 0