diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index b93d40738ea..12c5031628b 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -960,7 +960,8 @@ class AmazonConverseConfig(BaseConfig): ) -> bool: """Check if the Bedrock model takes native structured outputs (outputConfig.textFormat) for this request. - ``supports_native_structured_output`` in ``litellm.model_cost`` turns it on for every request, and + ``supports_native_structured_output`` or ``supports_bedrock_native_structured_output_by_default`` in + ``litellm.model_cost`` turns it on for every request, and ``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) also when Claude will think, which the json tool path would drop. When the request does not ask for thinking, a model that cannot force the json tool gets it unless that key is false. diff --git a/litellm/llms/bedrock/chat/native_output_fallback.py b/litellm/llms/bedrock/chat/native_output_fallback.py index 79742d0810c..7d5662e2024 100644 --- a/litellm/llms/bedrock/chat/native_output_fallback.py +++ b/litellm/llms/bedrock/chat/native_output_fallback.py @@ -9,7 +9,9 @@ from litellm.llms.bedrock.common_utils import ( NativeOutputFallback, bedrock_strict_tools_fallback, is_bedrock_native_output_rejection, + is_bedrock_stream_permission_denial, warn_bedrock_native_output_rejected, + warn_bedrock_native_output_stream_denied, warn_bedrock_strict_tools_dropped, ) from litellm.types.utils import ModelResponse @@ -25,22 +27,55 @@ NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | Non class _SelectedFallback: fallback: NativeOutputFallback native_output_retry: bool + buffered_native_output: bool = False _RetryFor: TypeAlias = Callable[[BaseLLMException], _SelectedFallback | None] +_NextRetryFor: TypeAlias = Callable[[BaseLLMException, _SelectedFallback], _SelectedFallback | None] -def _native_output_retry( - error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str +def _native_output_rejection_retry( + error: BaseLLMException, fallback: NativeOutputFallbackFactory, model: str ) -> NativeOutputFallback | None: - if fallback is None or not is_bedrock_native_output_rejection(error.status_code, error.message): - return None retry: Final = fallback() if retry is not None: warn_bedrock_native_output_rejected(model, error.message) return retry +def _buffered(retry: NativeOutputFallback) -> NativeOutputFallback: + return NativeOutputFallback( + optional_params={**retry.optional_params, "fake_stream": True}, litellm_params=retry.litellm_params + ) + + +def _buffered_stream_retry(fallback: NativeOutputFallbackFactory, model: str) -> NativeOutputFallback | None: + retry: Final = fallback() + if retry is None or retry.optional_params.get("fake_stream") is not True: + return None + warn_bedrock_native_output_stream_denied(model) + return retry + + +def _native_output_retry( + error: BaseLLMException, + fallback: NativeOutputFallbackFactory | None, + buffered_stream_fallback: NativeOutputFallbackFactory | None, + model: str, +) -> _SelectedFallback | None: + if fallback is not None and is_bedrock_native_output_rejection(error.status_code, error.message): + rejection_retry: Final = _native_output_rejection_retry(error, fallback, model) + return None if rejection_retry is None else _SelectedFallback(rejection_retry, native_output_retry=True) + if buffered_stream_fallback is not None and is_bedrock_stream_permission_denial(error.status_code, error.message): + stream_retry: Final = _buffered_stream_retry(buffered_stream_fallback, model) + return ( + None + if stream_retry is None + else _SelectedFallback(stream_retry, native_output_retry=True, buffered_native_output=True) + ) + return None + + def _strict_tools_retry( error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str ) -> NativeOutputFallback | None: @@ -55,12 +90,13 @@ def _strict_tools_retry( def _fallback_for( error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, + buffered_stream_fallback: NativeOutputFallbackFactory | None, strict_tools_fallback: NativeOutputFallbackFactory | None, model: str, ) -> _SelectedFallback | None: - native_output_retry: Final = _native_output_retry(error, fallback, model) + native_output_retry: Final = _native_output_retry(error, fallback, buffered_stream_fallback, model) if native_output_retry is not None: - return _SelectedFallback(native_output_retry, native_output_retry=True) + return native_output_retry strict_tools_retry: Final = _strict_tools_retry(error, strict_tools_fallback, model) if strict_tools_retry is None: return None @@ -87,12 +123,81 @@ def _strict_tools_retry_after_native_output_retry( return composed_retry +def _next_retry( + error: BaseLLMException, + retry: _SelectedFallback, + fallback: NativeOutputFallbackFactory | None, + strict_tools_fallback: NativeOutputFallbackFactory | None, + model: str, +) -> _SelectedFallback | None: + if ( + retry.buffered_native_output + and fallback is not None + and is_bedrock_native_output_rejection(error.status_code, error.message) + ): + rejection_retry: Final = _native_output_rejection_retry(error, fallback, model) + return ( + None if rejection_retry is None else _SelectedFallback(_buffered(rejection_retry), native_output_retry=True) + ) + composed_retry: Final = _strict_tools_retry_after_native_output_retry(error, retry, strict_tools_fallback, model) + return None if composed_retry is None else _SelectedFallback(composed_retry, native_output_retry=False) + + +def _attempt_or_next_retry( + attempt: NativeOutputAttempt, retry: _SelectedFallback, next_retry_for: _NextRetryFor +) -> NativeOutputAttemptResult | _SelectedFallback: + try: + return attempt(retry.fallback) + except BaseLLMException as error: + next_retry: Final = next_retry_for(error, retry) + if next_retry is None: + raise + return next_retry + + +def _attempt_retries( + attempt: NativeOutputAttempt, retry: _SelectedFallback, next_retry_for: _NextRetryFor +) -> NativeOutputAttemptResult: + after_first_retry: Final = _attempt_or_next_retry(attempt, retry, next_retry_for) + if not isinstance(after_first_retry, _SelectedFallback): + return after_first_retry + after_second_retry: Final = _attempt_or_next_retry(attempt, after_first_retry, next_retry_for) + if not isinstance(after_second_retry, _SelectedFallback): + return after_second_retry + return attempt(after_second_retry.fallback) + + +async def _await_or_next_retry( + attempt: NativeOutputAttempt, retry: _SelectedFallback, next_retry_for: _NextRetryFor +) -> NativeOutputResponse | _SelectedFallback: + try: + result: Final = attempt(retry.fallback) + return await result if isinstance(result, Coroutine) else result + except BaseLLMException as error: + next_retry: Final = next_retry_for(error, retry) + if next_retry is None: + raise + return next_retry + + +async def _await_retries( + attempt: NativeOutputAttempt, retry: _SelectedFallback, next_retry_for: _NextRetryFor +) -> NativeOutputResponse: + after_first_retry: Final = await _await_or_next_retry(attempt, retry, next_retry_for) + if not isinstance(after_first_retry, _SelectedFallback): + return after_first_retry + after_second_retry: Final = await _await_or_next_retry(attempt, after_first_retry, next_retry_for) + if not isinstance(after_second_retry, _SelectedFallback): + return after_second_retry + last_attempt: Final = attempt(after_second_retry.fallback) + return await last_attempt if isinstance(last_attempt, Coroutine) else last_attempt + + async def _await_with_native_output_fallback( first_attempt: Coroutine[object, object, NativeOutputResponse], attempt: NativeOutputAttempt, retry_for: _RetryFor, - strict_tools_fallback: NativeOutputFallbackFactory | None, - model: str, + next_retry_for: _NextRetryFor, ) -> NativeOutputResponse: try: return await first_attempt @@ -100,17 +205,7 @@ async def _await_with_native_output_fallback( retry: Final = retry_for(error) if retry is None: raise - try: - second_attempt: Final = attempt(retry.fallback) - return await second_attempt if isinstance(second_attempt, Coroutine) else second_attempt - except BaseLLMException as error: - composed_retry: Final = _strict_tools_retry_after_native_output_retry( - error, retry, strict_tools_fallback, model - ) - if composed_retry is None: - raise - third_attempt: Final = attempt(composed_retry) - return await third_attempt if isinstance(third_attempt, Coroutine) else third_attempt + return await _await_retries(attempt, retry, next_retry_for) def call_with_native_output_fallback( @@ -118,11 +213,19 @@ def call_with_native_output_fallback( fallback: NativeOutputFallbackFactory | None, model: str, strict_tools_fallback: NativeOutputFallbackFactory | None = None, + buffered_stream_fallback: NativeOutputFallbackFactory | None = None, ) -> NativeOutputAttemptResult: - if fallback is None and strict_tools_fallback is None: + if fallback is None and strict_tools_fallback is None and buffered_stream_fallback is None: return attempt(None) retry_for: Final = partial( - _fallback_for, fallback=fallback, strict_tools_fallback=strict_tools_fallback, model=model + _fallback_for, + fallback=fallback, + buffered_stream_fallback=buffered_stream_fallback, + strict_tools_fallback=strict_tools_fallback, + model=model, + ) + next_retry_for: Final = partial( + _next_retry, fallback=fallback, strict_tools_fallback=strict_tools_fallback, model=model ) try: first_attempt: Final = attempt(None) @@ -130,15 +233,7 @@ def call_with_native_output_fallback( retry: Final = retry_for(error) if retry is None: raise - try: - return attempt(retry.fallback) - except BaseLLMException as retry_error: - composed_retry: Final = _strict_tools_retry_after_native_output_retry( - retry_error, retry, strict_tools_fallback, model - ) - if composed_retry is None: - raise - return attempt(composed_retry) + return _attempt_retries(attempt, retry, next_retry_for) if isinstance(first_attempt, Coroutine): - return _await_with_native_output_fallback(first_attempt, attempt, retry_for, strict_tools_fallback, model) + return _await_with_native_output_fallback(first_attempt, attempt, retry_for, next_retry_for) return first_attempt diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index f0627c7362c..0f8694a7e46 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -297,9 +297,16 @@ def _bedrock_model_supports(model: str, key: str, custom_llm_provider: str = "be def _bedrock_native_output_enabled( model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None ) -> bool: - deployment_flag: Final = (model_info or {}).get("supports_native_structured_output") + deployment_info: Final = model_info or {} + deployment_flag: Final = deployment_info.get("supports_native_structured_output") if isinstance(deployment_flag, bool): return deployment_flag + return deployment_info.get("supports_bedrock_native_structured_output") is not False and ( + bedrock_model_always_takes_native_output(model, custom_llm_provider) + ) + + +def bedrock_model_always_takes_native_output(model: str, custom_llm_provider: str = "bedrock") -> bool: return _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) @@ -443,6 +450,18 @@ def warn_bedrock_native_output_rejected(model: str, error_text: str) -> None: ) +def is_bedrock_stream_permission_denial(status_code: int, error_text: str) -> bool: + return status_code == 403 and "bedrock:InvokeModelWithResponseStream" in error_text + + +def warn_bedrock_native_output_stream_denied(model: str) -> None: + verbose_logger.warning( + "Bedrock denied bedrock:InvokeModelWithResponseStream for model=%s, so litellm sent the structured output " + "request again as a buffered stream. Allow that action to stream it.", + model, + ) + + @dataclass(frozen=True, slots=True) class NativeOutputFallback: optional_params: Mapping[str, object] @@ -543,6 +562,22 @@ def json_object_or_none(value: object) -> Mapping[str, object] | None: BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES: Final = 64 +def _response_format_json_schema(response_format: object) -> Mapping[str, object] | None: + json_schema: Final = json_object_or_none((json_object_or_none(response_format) or {}).get("json_schema")) + return json_object_or_none((json_schema or {}).get("schema")) + + +def bedrock_batch_record_takes_native_output( + model: str, response_format: object, model_info: Mapping[str, object] | None = None +) -> bool: + schema: Final = _response_format_json_schema(response_format) + return _bedrock_native_output_enabled(model, "bedrock", model_info) and ( + schema is None + or schema_property_count(schema) <= BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES + or not _bedrock_model_supports(model, "supports_bedrock_native_structured_output_by_default") + ) + + def bedrock_cleaned_output_schema( model: str, schema: Mapping[str, object], custom_llm_provider: str = "bedrock" ) -> Mapping[str, object] | None: diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 4d137876890..ef54c85c131 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -71,6 +71,7 @@ from litellm.utils import get_llm_provider, get_optional_params from ..base_aws_llm import BaseAWSLLM from ..common_utils import ( BedrockError, + bedrock_batch_record_takes_native_output, deployment_model_info, merge_bedrock_aws_request_params, resolve_s3_encryption_key_id, @@ -167,6 +168,20 @@ _LITELLM_PARAMS_THE_MAPPER_TAKES: Final = frozenset({"allowed_openai_params"}) _MAPPED_PARAMS_THE_REQUEST_HANDLER_STRIPS: Final = frozenset({"json_mode"}) +_UPLOAD_PARAMS: Final = TypeAdapter(dict[str, object]) + + +def _upload_params(params: object) -> Mapping[str, object]: + return _UPLOAD_PARAMS.validate_python(params) if isinstance(params, Mapping) else {} + + +def _upload_deployment_model_info(litellm_params: object) -> Mapping[str, object]: + upload_params: Final = _upload_params(litellm_params) + return deployment_model_info(upload_params) or deployment_model_info( + _upload_params(upload_params.get("litellm_metadata")) + ) + + def _invoke_route_model(model: str) -> str: return f"invoke/{_strip_llm_routing_prefix(model).removeprefix('invoke/')}" @@ -1034,6 +1049,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): openai_request_body: Mapping[str, Any], model: str, provider: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> dict[str, object]: """ Transform OpenAI request body to Bedrock-compatible modelInput @@ -1055,7 +1071,15 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): ) config: Final = AmazonAnthropicClaudeConfig() - litellm_params: Final = without_bedrock_native_output_opt_in({}) + litellm_params: Final = without_bedrock_native_output_opt_in( + { + "model_info": { + "supports_native_structured_output": bedrock_batch_record_takes_native_output( + _invoke_route_model(model), optional_params.get("response_format"), model_info + ) + } + } + ) mapped_params = get_optional_params( model=_invoke_route_model(model), custom_llm_provider="bedrock", @@ -1150,25 +1174,32 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): ] def transform_batch_input_lines( - self, batch_input_lines: Sequence[_BatchInputLine], target_model: str = "" + self, + batch_input_lines: Sequence[_BatchInputLine], + target_model: str = "", + model_info: Mapping[str, object] | None = None, ) -> tuple[_BedrockBatchRecord, ...]: return tuple( - self._transform_batch_input_line(idx, batch_input_line, target_model) + self._transform_batch_input_line(idx, batch_input_line, target_model, model_info) for idx, batch_input_line in enumerate(batch_input_lines) ) def _transform_batch_input_line( - self, idx: int, batch_input_line: _BatchInputLine, target_model: str + self, idx: int, batch_input_line: _BatchInputLine, target_model: str, model_info: Mapping[str, object] | None ) -> _BedrockBatchRecord: try: - return self._transform_openai_batch_record(idx, batch_input_line.record, target_model) + return self._transform_openai_batch_record(idx, batch_input_line.record, target_model, model_info) except _BATCH_RECORD_CONVERSION_ERRORS as e: raise _batch_line_error( batch_input_line.line_number, f"could not be converted to a Bedrock batch record: {e}" ) from e def _transform_openai_batch_record( - self, idx: int, openai_record: _OpenAIBatchRecord, target_model: str + self, + idx: int, + openai_record: _OpenAIBatchRecord, + target_model: str, + model_info: Mapping[str, object] | None = None, ) -> _BedrockBatchRecord: import litellm @@ -1192,6 +1223,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): openai_request_body=self._transform_batch_body_to_chat_body(openai_body, record_kind), model=model_for_transform, provider=provider, + model_info=model_info, ) ) return _BedrockBatchRecord( @@ -1211,7 +1243,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): model: str, create_file_data: CreateFileRequest, optional_params: dict, - litellm_params: dict, + litellm_params: dict[str, object], ) -> bytes | str | dict: """ Transform file request and return a pre-signed request for S3. @@ -1237,7 +1269,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): litellm_params_model: Final = litellm_params.get("model") target_model: Final = model or (litellm_params_model if isinstance(litellm_params_model, str) else "") bedrock_jsonl_content: Final = self.transform_batch_input_lines( - batch_input_lines, target_model=target_model + batch_input_lines, target_model=target_model, model_info=_upload_deployment_model_info(litellm_params) ) file_content = "\n".join(json.dumps(item) for item in bedrock_jsonl_content) elif isinstance(extracted_file_data_content, bytes): diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 169c21cf635..df82891f789 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -897,7 +897,7 @@ class BaseLLMHTTPHandler: fake_stream = ( fake_stream - or optional_params.pop("fake_stream", False) + or optional_params.pop("fake_stream", False) is True or provider_config.should_fake_stream(model=model, custom_llm_provider=custom_llm_provider, stream=stream) ) @@ -920,7 +920,7 @@ class BaseLLMHTTPHandler: api_key=api_key, model=model, optional_params=optional_params, - stream=stream, + stream=False if fake_stream else stream, litellm_params=litellm_params, ) @@ -1056,7 +1056,7 @@ class BaseLLMHTTPHandler: provider_config=provider_config, fake_stream=fake_stream, ) - if provider_config.has_custom_stream_wrapper is True: + if provider_config.has_custom_stream_wrapper is True and not fake_stream: return provider_config.get_sync_custom_stream_wrapper( model=model, custom_llm_provider=custom_llm_provider, @@ -1225,7 +1225,7 @@ class BaseLLMHTTPHandler: json_mode: bool | None = None, signed_json_body: bytes | None = None, ): - if provider_config.has_custom_stream_wrapper is True: + if provider_config.has_custom_stream_wrapper is True and not fake_stream: return await provider_config.get_async_custom_stream_wrapper( model=model, custom_llm_provider=custom_llm_provider, diff --git a/litellm/main.py b/litellm/main.py index 6b9964a450f..bdb35823757 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4393,6 +4393,14 @@ def _converse_native_output_fallback( return None if AmazonConverseConfig.sends_native_text_format(fallback.optional_params) else fallback +def _buffered_stream_fallback( + first_attempt_params: Mapping[str, object], first_attempt_litellm_params: Mapping[str, object] +) -> NativeOutputFallback: + return NativeOutputFallback( + optional_params={**first_attempt_params, "fake_stream": True}, litellm_params=first_attempt_litellm_params + ) + + def _native_output_attempt_timeout( ctx: CompletionDispatchContext, fallback: NativeOutputFallback | None ) -> float | str | httpx.Timeout | None: @@ -4496,11 +4504,14 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu ) converse_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params) + converse_uses_native_output: Final = AmazonConverseConfig().uses_native_output( + converse_model, converse_params, litellm_params + ) response = call_with_native_output_fallback( converse_attempt, ( partial(_converse_native_output_fallback, ctx, converse_params, dict(litellm_params)) - if AmazonConverseConfig().uses_native_output(converse_model, converse_params, litellm_params) + if converse_uses_native_output else None ), converse_model, @@ -4509,6 +4520,11 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu if drops_unsupported_params(litellm_params.get("drop_params")) else None ), + buffered_stream_fallback=( + partial(_buffered_stream_fallback, converse_params, dict(litellm_params)) + if converse_uses_native_output + else None + ), ) elif bedrock_route == "converse_like": model = model.replace("converse_like/", "") @@ -4551,15 +4567,20 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu ) invoke_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params) + invoke_uses_native_output: Final = isinstance( + provider_config, AmazonAnthropicClaudeConfig + ) and provider_config.uses_native_output(model, invoke_params, litellm_params) response = call_with_native_output_fallback( invoke_attempt, - ( - partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params)) - if isinstance(provider_config, AmazonAnthropicClaudeConfig) - and provider_config.uses_native_output(model, invoke_params, litellm_params) + partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params)) + if invoke_uses_native_output + else None, + model, + buffered_stream_fallback=( + partial(_buffered_stream_fallback, invoke_params, dict(litellm_params)) + if invoke_uses_native_output else None ), - model, ) return response diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4e81afb0ea6..d9d996208d8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1961,6 +1961,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2041,6 +2042,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2116,6 +2118,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2242,7 +2245,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2287,6 +2291,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2331,7 +2336,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2376,6 +2382,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2420,7 +2427,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2465,6 +2473,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2504,7 +2513,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2549,6 +2559,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2593,7 +2604,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2633,6 +2645,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2715,7 +2728,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2759,7 +2773,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2803,7 +2818,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2847,7 +2863,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2891,7 +2908,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3023,6 +3041,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3068,6 +3087,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3113,6 +3133,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3158,6 +3179,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3203,6 +3225,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -75549,7 +75572,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -75589,7 +75613,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -75629,6 +75654,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -75702,6 +75728,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77417,6 +77444,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77474,6 +77502,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77651,6 +77680,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77703,6 +77733,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77750,6 +77781,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77863,6 +77895,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f5369bc34b0..7f48d77ff09 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -221,7 +221,12 @@ from litellm.types.utils import ( is_server_derived_pricing_key, pricing_override_fields, ) -from litellm.utils import acredential_updates, cost_map_omits_token_price, with_named_credential +from litellm.utils import ( + acredential_updates, + cost_map_omits_token_price, + native_structured_output_support, + with_named_credential, +) if TYPE_CHECKING: from aiohttp import ClientSession @@ -16733,7 +16738,7 @@ def _enrich_model_info_with_litellm_data( stamped_model_info: Final = MappingProxyType( {**model_info, **_pricing_override_stamps(model_info, model.get("litellm_params") or MappingProxyType({}))} ) - model["model_info"] = { + merged_model_info: Final = { **stamped_model_info, **{ k: None if unpriced and k in ("input_cost_per_token", "output_cost_per_token") else v @@ -16741,6 +16746,18 @@ def _enrich_model_info_with_litellm_data( if k not in stamped_model_info or (stamped_model_info[k] is None and k in discovered_model_info) }, } + native_output_info: Final = { + **merged_model_info, + "supports_bedrock_native_structured_output_by_default": litellm_model_info.get( + "supports_bedrock_native_structured_output_by_default" + ), + } + model["model_info"] = ( + {**merged_model_info, "supports_native_structured_output": native_structured_output_support(native_output_info)} + if "supports_native_structured_output" in merged_model_info + and "supports_native_structured_output" not in stamped_model_info + else merged_model_info + ) # don't return the api key / vertex credentials # don't return the llm credentials model = remove_sensitive_info_from_deployment(model, excluded_keys={"litellm_credential_name"}) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5a14dc95e7c..b082198d3f2 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -217,6 +217,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_native_streaming: bool | None supports_native_structured_output: bool | None supports_bedrock_native_structured_output: ReadOnly[bool | None] + supports_bedrock_native_structured_output_by_default: ReadOnly[bool | None] supports_parallel_function_calling: bool | None supports_web_search: bool | None supports_reasoning: bool | None @@ -4303,6 +4304,7 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( "supports_regex_lookaround", "supports_assistant_prefill", "supports_bedrock_native_structured_output", + "supports_bedrock_native_structured_output_by_default", "supports_per_turn_control", "supports_pdf_input", "supports_continuation", diff --git a/litellm/utils.py b/litellm/utils.py index 8944ecb5648..13f22ddc36e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4167,6 +4167,13 @@ def _registration_base( return _RegistrationWrite(key=key, entry=dict(fallback.entry)) +def native_structured_output_support(model_info: Mapping[str, object]) -> bool | None: + if model_info.get("supports_bedrock_native_structured_output_by_default"): + return model_info.get("supports_bedrock_native_structured_output") is not False + declared: Final = model_info.get("supports_native_structured_output") + return declared if isinstance(declared, bool) else None + + def _base_fields_replaced_by(value: Mapping[str, object]) -> frozenset[str]: return ( frozenset({"explicit_cache_read_input_token_cost"}) @@ -7587,10 +7594,13 @@ def get_model_info_helper( supports_pdf_input=_model_info.get("supports_pdf_input", None), supports_embedding_image_input=_model_info.get("supports_embedding_image_input", None), supports_native_streaming=_model_info.get("supports_native_streaming", None), - supports_native_structured_output=_model_info.get("supports_native_structured_output", None), + supports_native_structured_output=native_structured_output_support(_model_info), supports_bedrock_native_structured_output=_model_info.get( "supports_bedrock_native_structured_output", None ), + supports_bedrock_native_structured_output_by_default=_model_info.get( + "supports_bedrock_native_structured_output_by_default", None + ), supports_web_search=_model_info.get("supports_web_search", None), supports_url_context=_model_info.get("supports_url_context", None), supports_continuation=_model_info.get("supports_continuation", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4e81afb0ea6..d9d996208d8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1961,6 +1961,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2041,6 +2042,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2116,6 +2118,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2242,7 +2245,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2287,6 +2291,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2331,7 +2336,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2376,6 +2382,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2420,7 +2427,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2465,6 +2473,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2504,7 +2513,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2549,6 +2559,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2593,7 +2604,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2633,6 +2645,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2715,7 +2728,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2759,7 +2773,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2803,7 +2818,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2847,7 +2863,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2891,7 +2908,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3023,6 +3041,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3068,6 +3087,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3113,6 +3133,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3158,6 +3179,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3203,6 +3225,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -75549,7 +75572,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -75589,7 +75613,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -75629,6 +75654,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -75702,6 +75728,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77417,6 +77444,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77474,6 +77502,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77651,6 +77680,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77703,6 +77733,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77750,6 +77781,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77863,6 +77895,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, diff --git a/tests/integration/providers/test_bedrock_batch_files_wire.py b/tests/integration/providers/test_bedrock_batch_files_wire.py index 823bcda7e06..1657895109d 100644 --- a/tests/integration/providers/test_bedrock_batch_files_wire.py +++ b/tests/integration/providers/test_bedrock_batch_files_wire.py @@ -88,10 +88,12 @@ _SCHEMA: Final = { @pytest.mark.parametrize( - "backend", ["us.anthropic.claude-haiku-5-5", "us.anthropic.claude-opus-4-8"], ids=["haiku_5_5", "opus_4_8"] + ("backend", "native_output"), + [("us.anthropic.claude-sonnet-5-5", False), ("us.anthropic.claude-haiku-5-5", True)], + ids=["opt_in_native_output_model", "native_output_model"], ) -def test_json_schema_batch_record_with_thinking_uploads_the_json_tool_and_no_native_output( - gateway: Gateway, backend: str +def test_json_schema_batch_record_with_thinking_uploads_native_output_only_on_models_that_always_take_it( + gateway: Gateway, backend: str, native_output: bool ) -> None: record: Final = { "custom_id": "schema-record", @@ -125,7 +127,13 @@ def test_json_schema_batch_record_with_thinking_uploads_the_json_tool_and_no_nat (upload,) = wire.drain() (stored,) = tuple(JSON_OBJECT.validate_json(line) for line in upload.body.decode().splitlines() if line.strip()) model_input: Final = object_value(stored["modelInput"]) - assert "format" not in object_value(model_input.get("output_config", {})), model_input - (tool,) = (object_value(entry) for entry in list_value(model_input["tools"])) + output_format: Final = object_value(model_input.get("output_config", {})).get("format") + tools: Final = tuple(object_value(entry) for entry in list_value(model_input.get("tools", []))) + if native_output: + assert object_value(object_value(output_format)["schema"])["properties"] == _SCHEMA["properties"], model_input + assert not tools, model_input + return + assert output_format is None, model_input + (tool,) = tools assert tool["name"] == _JSON_TOOL, model_input assert object_value(tool["input_schema"])["properties"] == _SCHEMA["properties"], model_input diff --git a/tests/integration/providers/test_bedrock_native_structured_output_wire.py b/tests/integration/providers/test_bedrock_native_structured_output_wire.py index 2502bd90bd6..68942ed295b 100644 --- a/tests/integration/providers/test_bedrock_native_structured_output_wire.py +++ b/tests/integration/providers/test_bedrock_native_structured_output_wire.py @@ -18,9 +18,10 @@ from pydantic import JsonValue, TypeAdapter _HAIKU: Final = "us.anthropic.claude-haiku-5-5" _FABLE: Final = "us.anthropic.claude-fable-5" _SONNET: Final = "us.anthropic.claude-sonnet-4-6" -_OPUS: Final = "us.anthropic.claude-opus-5-5" -_OPT_IN: Final = _OPUS -_UNFORCEABLE_TOOL: Final = _OPUS +_NEITHER_KEY: Final = "us.anthropic.claude-opus-4-7" +_OPT_IN: Final = _NEITHER_KEY +_BEDROCK_KEY_ONLY: Final = "us.anthropic.claude-sonnet-5-5" +_UNFORCEABLE_TOOL: Final = "anthropic.claude-opus-5-5" _OPUS_4_8: Final = "us.anthropic.claude-opus-4-8" _DEEPSEEK: Final = "deepseek.v3.2" _AWS: Final[dict[str, JsonValue]] = { @@ -683,7 +684,7 @@ def test_a_model_that_cannot_force_the_json_tool_retries_a_rejected_native_outpu @pytest.mark.parametrize( ("backend", "model_info", "native"), [ - (_OPUS, None, False), + (_NEITHER_KEY, None, False), (_OPT_IN, {"supports_bedrock_native_structured_output": True}, True), (_OPUS_4_8, {"supports_bedrock_native_structured_output": False}, False), ], @@ -897,7 +898,7 @@ def test_messages_on_bedrock_invoke_follows_the_bedrock_native_key_without_think ) -> None: marker, marker_hex = _marker() with wire_server(_bedrock) as wire, gateway.scenario() as scenario: - model: Final = _deployment(scenario, wire, "invoke", _HAIKU) + model: Final = _deployment(scenario, wire, "invoke", _BEDROCK_KEY_ONLY) response: Final = gateway.request( "POST", "/v1/messages", @@ -945,9 +946,9 @@ def test_malformed_schema_on_a_native_model_reaches_bedrock_once_without_a_proxy ("route", "backend", "thinking", "schema", "native"), [ ("converse", _SONNET, False, _NUMERIC_FORMATS_SCHEMA, True), - ("invoke", _HAIKU, True, _NUMERIC_FORMATS_SCHEMA, True), + ("invoke", _BEDROCK_KEY_ONLY, True, _NUMERIC_FORMATS_SCHEMA, True), ("converse", _SONNET, False, _WIDE_SCHEMA, True), - ("converse", _HAIKU, True, _WIDE_SCHEMA, False), + ("converse", _BEDROCK_KEY_ONLY, True, _WIDE_SCHEMA, False), ], ids=["numeric_formats_old_key", "numeric_formats_new_key_invoke", "wide_old_key", "wide_new_key"], ) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 146a6a3ff1c..0a9dc0ab3cf 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -496,7 +496,7 @@ def declare_mid_conversation_system(monkeypatch: pytest.MonkeyPatch) -> Callable return declare -NativeStructuredOutputKey: TypeAlias = Literal["legacy", "bedrock"] | None +NativeStructuredOutputKey: TypeAlias = Literal["legacy", "bedrock", "default"] | None @pytest.fixture @@ -506,7 +506,8 @@ def set_native_structured_output( def set_flags(model_cost_key: str, native_key: NativeStructuredOutputKey) -> None: entry: Final = litellm.model_cost[model_cost_key] monkeypatch.setitem(entry, "supports_native_structured_output", native_key == "legacy") - monkeypatch.setitem(entry, "supports_bedrock_native_structured_output", native_key == "bedrock") + monkeypatch.setitem(entry, "supports_bedrock_native_structured_output", native_key in ("bedrock", "default")) + monkeypatch.setitem(entry, "supports_bedrock_native_structured_output_by_default", native_key == "default") litellm.get_model_info.cache_clear() return set_flags diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py index 053574d7dd6..4f17f8b10e1 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py @@ -570,6 +570,7 @@ def _invoke_params_for_schema( ("us.anthropic.claude-opus-4-8", "bedrock", {"thinking": {"type": "adaptive"}}, True), ("us.anthropic.claude-opus-4-8", "bedrock", {"reasoning_effort": "low"}, True), ("us.anthropic.claude-haiku-5-5", "bedrock", {"thinking": {"type": "disabled"}}, False), + ("us.anthropic.claude-haiku-5-5", "default", {"thinking": {"type": "disabled"}}, True), ], ids=[ "legacy-key", @@ -578,6 +579,7 @@ def _invoke_params_for_schema( "bedrock-key-adaptive", "bedrock-key-effort", "bedrock-key-thinking-disabled", + "default-key-thinking-disabled", ], ) def test_bedrock_native_output_key_applies_only_when_the_model_will_think( @@ -2047,9 +2049,9 @@ def _invoke_chat_params( ) @pytest.mark.parametrize("model", ["us.anthropic.claude-opus-4-8", "global.anthropic.claude-sonnet-5"]) def test_invoke_uses_opt_in_native_output_when_the_opt_in_drove_native_output( - local_model_cost_map, model, thinking_params, monkeypatch + set_native_structured_output, model, thinking_params ): - monkeypatch.setitem(litellm.model_cost[model], "supports_native_structured_output", False) + set_native_structured_output(model, "bedrock") optional_params: Final = _invoke_chat_params( model, None, {"response_format": _CITY_RESPONSE_FORMAT, **thinking_params} ) @@ -2084,8 +2086,12 @@ def test_invoke_uses_opt_in_native_output_is_false_when_native_output_did_not_co ) -def test_invoke_uses_default_native_output_for_schema_retry(local_model_cost_map: None) -> None: - model: Final = "us.anthropic.claude-opus-5" +@pytest.mark.parametrize( + "model", + ["us.anthropic.claude-opus-5", "us.anthropic.claude-sonnet-5"], + ids=["opus-5", "sonnet-5"], +) +def test_invoke_uses_default_native_output_for_schema_retry(local_model_cost_map: None, model: str) -> None: optional_params: Final = _invoke_chat_params( model, None, {"response_format": _CITY_RESPONSE_FORMAT, "thinking": {"type": "disabled"}} ) diff --git a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py index 7d1b502df2b..d43cc771185 100644 --- a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py +++ b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py @@ -599,6 +599,139 @@ async def test_native_output_retry_is_bounded_by_the_request_timeout_not_the_str assert _sent_paths(respx_mock) == ["converse-stream", "converse"] +# Bedrock's 403 for a role without the streaming action, seen live on us.anthropic.claude-sonnet-5 on 2026-11-06 +_STREAM_PERMISSION_DENIAL: Final = json.dumps( + { + "Message": "User: arn:aws:sts::123456789012:assumed-role/invoke-only/qa is not authorized to perform: " + "bedrock:InvokeModelWithResponseStream on resource: arn:aws:bedrock:us-east-1:123456789012:inference-profile/" + "us.anthropic.claude-opus-5-5 because no identity-based policy allows the " + "bedrock:InvokeModelWithResponseStream action" + } +).encode() + + +def _denies_streaming_then_answers(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/converse-stream"): + return httpx.Response(403, content=_STREAM_PERMISSION_DENIAL, request=request) + return httpx.Response(200, content=CONVERSE_ANSWER, headers={"content-type": "application/json"}, request=request) + + +@pytest.mark.asyncio +async def test_a_denied_native_output_stream_is_sent_again_buffered_with_native_output( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post(url__regex=_BEDROCK_RUNTIME_URL).mock(side_effect=_denies_streaming_then_answers) + router: Final = _bedrock_router( + "bedrock/converse/us.anthropic.claude-opus-5-5", timeout=60, stream_timeout=STREAM_TIMEOUT_SECONDS + ) + + response: Final = await router.acompletion( + model="bedrock", + messages=[{"role": "user", "content": "hi"}], + stream=True, + thinking={"type": "disabled"}, + response_format={"type": "json_schema", "json_schema": {"name": "answer", "schema": _ANSWER_SCHEMA}}, + ) + + assert "".join([chunk.choices[0].delta.content or "" async for chunk in response]) == "hi" + assert _sent_paths(respx_mock) == ["converse-stream", "converse"] + retry_body: Final = json.loads(respx_mock.calls[1].request.content) + assert "textFormat" in retry_body.get("outputConfig", {}) and "toolConfig" not in retry_body + + +_INVOKE_ANSWER: Final = json.dumps( + { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-opus-5-5", + "content": [{"type": "text", "text": '{"answer": "hi"}'}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 5}, + } +).encode() + + +def _denies_invoke_streaming_then_answers(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/invoke-with-response-stream"): + return httpx.Response(403, content=_STREAM_PERMISSION_DENIAL, request=request) + return httpx.Response(200, content=_INVOKE_ANSWER, headers={"content-type": "application/json"}, request=request) + + +@pytest.mark.asyncio +async def test_a_denied_native_output_stream_on_the_invoke_route_is_sent_again_buffered_to_invoke( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post(url__regex=_BEDROCK_RUNTIME_URL).mock(side_effect=_denies_invoke_streaming_then_answers) + router: Final = _bedrock_router( + "bedrock/invoke/us.anthropic.claude-opus-5-5", timeout=60, stream_timeout=STREAM_TIMEOUT_SECONDS + ) + + response: Final = await router.acompletion( + model="bedrock", + messages=[{"role": "user", "content": "hi"}], + stream=True, + thinking={"type": "disabled"}, + response_format={"type": "json_schema", "json_schema": {"name": "answer", "schema": _ANSWER_SCHEMA}}, + ) + + assert "".join([chunk.choices[0].delta.content or "" async for chunk in response]) == '{"answer": "hi"}' + assert _sent_paths(respx_mock) == ["invoke-with-response-stream", "invoke"] + retry_body: Final = json.loads(respx_mock.calls[1].request.content) + assert "format" in retry_body.get("output_config", {}) and "tools" not in retry_body + + +_INVOKE_JSON_TOOL_ANSWER: Final = json.dumps( + { + "id": "msg_2", + "type": "message", + "role": "assistant", + "model": "claude-opus-5-5", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "json_tool_call", "input": {"answer": "hi"}}], + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 5}, + } +).encode() + + +def _denies_invoke_streaming_and_rejects_native_output(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/invoke-with-response-stream"): + return httpx.Response(403, content=_STREAM_PERMISSION_DENIAL, request=request) + if "format" in json.loads(request.content).get("output_config", {}): + return httpx.Response(400, content=_NATIVE_OUTPUT_REJECTION, request=request) + return httpx.Response( + 200, content=_INVOKE_JSON_TOOL_ANSWER, headers={"content-type": "application/json"}, request=request + ) + + +@pytest.mark.asyncio +async def test_a_rejected_schema_after_a_denied_invoke_stream_falls_back_to_the_json_tool_still_buffered( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post(url__regex=_BEDROCK_RUNTIME_URL).mock(side_effect=_denies_invoke_streaming_and_rejects_native_output) + router: Final = _bedrock_router( + "bedrock/invoke/us.anthropic.claude-opus-5-5", timeout=60, stream_timeout=STREAM_TIMEOUT_SECONDS + ) + + response: Final = await router.acompletion( + model="bedrock", + messages=[{"role": "user", "content": "hi"}], + stream=True, + thinking={"type": "disabled"}, + response_format={"type": "json_schema", "json_schema": {"name": "answer", "schema": _ANSWER_SCHEMA}}, + ) + + assert "".join([chunk.choices[0].delta.content or "" async for chunk in response]) == '{"answer": "hi"}' + assert _sent_paths(respx_mock) == ["invoke-with-response-stream", "invoke", "invoke"] + json_tool_body: Final = json.loads(respx_mock.calls[2].request.content) + assert "format" not in json_tool_body.get("output_config", {}) and "tools" in json_tool_body + + @pytest.mark.asyncio async def test_failed_native_output_retry_reports_the_request_timeout_it_was_bounded_by( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index b37d6977c10..4dcf312737d 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -5225,9 +5225,9 @@ def test_a_model_that_cannot_force_the_json_tool_gets_native_output_without_thin ("model", "thinking_params", "native"), [ ("us.anthropic.claude-opus-5-5", None, True), - ("us.anthropic.claude-opus-5-5", {"thinking": {"type": "adaptive"}}, False), + ("us.anthropic.claude-opus-5-5", {"thinking": {"type": "adaptive"}}, True), ("global.anthropic.claude-fable-5-1", None, True), - ("global.anthropic.claude-fable-5-1", {"reasoning_effort": "low"}, False), + ("global.anthropic.claude-fable-5-1", {"reasoning_effort": "low"}, True), ("us-gov.anthropic.claude-opus-5-5", None, False), ("us-gov.anthropic.claude-fable-5-1", None, False), ("us-gov-west-1/anthropic.claude-opus-5-5", None, False), @@ -5235,7 +5235,7 @@ def test_a_model_that_cannot_force_the_json_tool_gets_native_output_without_thin ("us-east-1/anthropic.claude-opus-5-5", None, True), ], ) -def test_models_whose_thinking_stops_under_native_output_keep_the_json_tool_only_when_thinking( +def test_native_output_follows_model_cost_map_for_bedrock_profiles( model: str, thinking_params: Mapping[str, object] | None, native: bool ) -> None: _, request = _converse_request_for_json_schema(model, False, thinking_params) @@ -5248,6 +5248,17 @@ def test_models_whose_thinking_stops_under_native_output_keep_the_json_tool_only @pytest.mark.parametrize( ("model", "thinking_params", "native"), [ + ("us.anthropic.claude-opus-5-5", {"thinking": {"type": "disabled"}}, True), + ("global.anthropic.claude-fable-5-1", {"thinking": {"type": "adaptive"}}, True), + ("global.anthropic.claude-fable-5-1", {"thinking": {"type": "disabled"}}, True), + ("us.anthropic.claude-sonnet-5", None, True), + ("us.anthropic.claude-sonnet-5", {"thinking": {"type": "adaptive"}}, True), + ("us.anthropic.claude-sonnet-5", {"thinking": {"type": "disabled"}}, True), + ("us.anthropic.claude-haiku-5-5", None, True), + ("us.anthropic.claude-haiku-5-5", {"thinking": {"type": "adaptive"}}, True), + ("us.anthropic.claude-haiku-5-5", {"thinking": {"type": "disabled"}}, True), + ("us-gov.anthropic.claude-sonnet-5", None, False), + ("us-gov.anthropic.claude-haiku-5-5", None, False), ("us.anthropic.claude-opus-5", None, True), ("us.anthropic.claude-opus-5", {"thinking": {"type": "adaptive"}}, True), ("us.anthropic.claude-opus-5", {"thinking": {"type": "disabled"}}, True), @@ -5258,6 +5269,17 @@ def test_models_whose_thinking_stops_under_native_output_keep_the_json_tool_only ("us-gov.anthropic.claude-opus-4-8", None, False), ], ids=[ + "opus-5-5-disabled", + "fable-5-1-adaptive", + "fable-5-1-disabled", + "sonnet-5-unset", + "sonnet-5-adaptive", + "sonnet-5-disabled", + "haiku-5-5-unset", + "haiku-5-5-adaptive", + "haiku-5-5-disabled", + "sonnet-5-us-gov", + "haiku-5-5-us-gov", "opus-5-unset", "opus-5-adaptive", "opus-5-disabled", @@ -5268,7 +5290,7 @@ def test_models_whose_thinking_stops_under_native_output_keep_the_json_tool_only "opus-4-8-us-gov", ], ) -def test_opus_5_and_4_8_cost_map_defaults_use_native_output_across_thinking_modes( +def test_model_cost_native_output_defaults_hold_across_thinking_modes( model: str, thinking_params: Mapping[str, object] | None, native: bool ) -> None: _, request = _converse_request_for_json_schema(model, False, thinking_params) @@ -5282,6 +5304,30 @@ def test_opus_5_and_4_8_cost_map_defaults_use_native_output_across_thinking_mode assert [tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]] == ["json_tool_call"] +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + ("model", "thinking_params"), + [ + ("us.anthropic.claude-opus-5-5", {"thinking": {"type": "adaptive"}}), + ("global.anthropic.claude-fable-5-1", {"thinking": {"type": "adaptive"}}), + ("us.anthropic.claude-sonnet-5", {"thinking": {"type": "disabled"}}), + ("us.anthropic.claude-haiku-5-5", None), + ("us.anthropic.claude-opus-5", None), + ], +) +@pytest.mark.parametrize("stream", [False, True]) +def test_deployment_opt_out_keeps_the_json_tool_on_models_that_default_to_native_output( + model: str, thinking_params: Mapping[str, object] | None, stream: bool +) -> None: + optional_params, request = _converse_request_for_json_schema( + model, stream, thinking_params, model_info={"supports_bedrock_native_structured_output": False} + ) + + assert "outputConfig" not in request + assert [tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]] == ["json_tool_call"] + assert optional_params.get("fake_stream", False) is stream + + def _converse_capital_answer(request: httpx.Request) -> httpx.Response: return httpx.Response( 200, @@ -5295,9 +5341,7 @@ def _converse_capital_answer(request: httpx.Request) -> httpx.Response: @pytest.mark.usefixtures("local_model_cost_map") -@pytest.mark.parametrize( - "model", ["us.anthropic.claude-opus-5-5", "global.anthropic.claude-sonnet-5-5", "global.anthropic.claude-fable-5-1"] -) +@pytest.mark.parametrize("model", ["global.anthropic.claude-sonnet-5-5"]) def test_streamed_json_schema_on_a_model_that_cannot_force_a_tool_still_calls_converse_without_streaming( model: str, ) -> None: @@ -5610,7 +5654,9 @@ def test_schema_past_a_native_output_size_limit_keeps_the_tool_path( @pytest.mark.parametrize( - ("native_key", "native_past_limit"), [("legacy", True), ("bedrock", False)], ids=["legacy-key", "bedrock-key"] + ("native_key", "native_past_limit"), + [("legacy", True), ("default", True), ("bedrock", False)], + ids=["legacy-key", "default-key", "bedrock-key"], ) def test_field_count_limit_sends_the_json_tool_only_on_the_bedrock_native_key( set_native_structured_output, native_key, native_past_limit @@ -10335,7 +10381,6 @@ def test_converse_reserves_the_tool_cache_point_within_the_cap(tool_cache_params assert request["toolConfig"]["tools"][-1] == cache_point - @pytest.mark.parametrize( "unsent_tool_params", [ @@ -10385,6 +10430,7 @@ def test_converse_leaves_the_cap_to_markers_when_a_marked_tool_is_not_sent( ] assert cache_point not in request.get("toolConfig", {}).get("tools", []) + def _converse_tool_turn(*result_parts: dict[str, object]) -> list[dict[str, object]]: return [ { @@ -10506,7 +10552,9 @@ def test_converse_translates_prompt_cache_breakpoint_on_an_unmapped_arn_only_whe messages=[ { "role": "user", - "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}}], + "content": [ + {"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}} + ], } ], optional_params={}, @@ -10518,7 +10566,6 @@ def test_converse_translates_prompt_cache_breakpoint_on_an_unmapped_arn_only_whe assert user_content(unflagged_arn, {"ttl": "30m"}) == [{"text": "Hello"}] - def test_converse_prompt_cache_breakpoint_follows_the_arn_flag_over_its_base_model(local_model_cost_map): from litellm import Router @@ -10545,7 +10592,9 @@ def test_converse_prompt_cache_breakpoint_follows_the_arn_flag_over_its_base_mod messages=[ { "role": "user", - "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}}], + "content": [ + {"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}} + ], } ], optional_params={}, @@ -10633,7 +10682,9 @@ def test_converse_prompt_cache_breakpoint_on_guarded_user_text_keeps_its_cache_p def test_converse_guarded_user_text_keeps_no_cache_point_on_a_model_not_known_to_take_one(local_model_cost_map): request: Final = litellm.AmazonConverseConfig()._transform_request( model="arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/unmapped1", - messages=[{"role": "user", "content": [{"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}]}], + messages=[ + {"role": "user", "content": [{"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}]} + ], optional_params={"guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "DRAFT"}}, litellm_params={}, ) @@ -11638,9 +11689,9 @@ _OUTPUT_CONFIG_FORMAT: Final = { ) @pytest.mark.parametrize("model", ["us.anthropic.claude-opus-4-8", "global.anthropic.claude-sonnet-5"]) def test_uses_opt_in_native_output_when_the_opt_in_drove_native_output( - local_model_cost_map, monkeypatch, model, request_params + set_native_structured_output, model, request_params ): - monkeypatch.setitem(litellm.model_cost[model], "supports_native_structured_output", False) + set_native_structured_output(model, "bedrock") optional_params: Final = _converse_params(model, None, request_params) assert AmazonConverseConfig().uses_opt_in_native_output(model, optional_params, {"model_info": {"id": "d1"}}) @@ -11672,9 +11723,9 @@ def test_uses_opt_in_native_output_is_false_when_native_output_did_not_come_from def test_uses_opt_in_native_output_reads_the_base_model_of_an_application_inference_profile( - local_model_cost_map, monkeypatch + set_native_structured_output, ): - monkeypatch.setitem(litellm.model_cost["us.anthropic.claude-opus-4-8"], "supports_native_structured_output", False) + set_native_structured_output("us.anthropic.claude-opus-4-8", "bedrock") profile: Final = "arn:aws:bedrock:us-east-1:000000000000:application-inference-profile/profile-id" litellm_params: Final = {"base_model": "bedrock/us.anthropic.claude-opus-4-8"} optional_params: Final = _converse_params( @@ -11768,8 +11819,9 @@ def test_a_refused_native_schema_is_sent_again_only_when_the_retry_drops_native_ @pytest.mark.respx(assert_all_called=True) +@pytest.mark.parametrize("model", ["us.anthropic.claude-sonnet-5", "us.anthropic.claude-opus-5"]) def test_a_default_native_schema_rejection_retries_with_the_json_tool( - respx_mock: respx.Router, local_model_cost_map: None + respx_mock: respx.Router, local_model_cost_map: None, model: str ) -> None: route: Final = respx_mock.post(url__regex=r"/converse$").mock( return_value=httpx.Response(400, json={"message": _BEDROCK_SCHEMA_REFUSAL}) @@ -11777,7 +11829,7 @@ def test_a_default_native_schema_rejection_retries_with_the_json_tool( with pytest.raises(litellm.BadRequestError, match=re.escape(_BEDROCK_SCHEMA_REFUSAL)): completion( - model="bedrock/converse/us.anthropic.claude-opus-5", + model=f"bedrock/converse/{model}", messages=[{"role": "user", "content": "What is the capital of France?"}], response_format=CAPITAL_RESPONSE_FORMAT, thinking={"type": "disabled"}, @@ -11938,7 +11990,9 @@ def test_converse_sends_no_tools_when_drop_params_drops_every_hosted_mcp_tool(mo def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: return tuple( - r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() ) @@ -12102,9 +12156,15 @@ def test_converse_sends_only_the_header_betas_when_header_and_body_both_carry_be def _five_cache_hint_conversation() -> list[dict[str, object]]: return [ - {"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]}, + { + "role": "system", + "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}], + }, {"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]}, - {"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]}, + { + "role": "assistant", + "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}], + }, {"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": "answer 2"}, {"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]}, @@ -12121,7 +12181,10 @@ def _five_cache_hint_conversation() -> list[dict[str, object]]: ], ) def test_converse_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params( - drop_params: bool | str, expected_system_cache_points: int, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None + drop_params: bool | str, + expected_system_cache_points: int, + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, ) -> None: monkeypatch.setattr(litellm, "drop_params", False) messages: Final = _five_cache_hint_conversation() diff --git a/tests/unit/llms/bedrock/chat/test_native_output_fallback.py b/tests/unit/llms/bedrock/chat/test_native_output_fallback.py index 81832d7d8bf..c87451d9577 100644 --- a/tests/unit/llms/bedrock/chat/test_native_output_fallback.py +++ b/tests/unit/llms/bedrock/chat/test_native_output_fallback.py @@ -149,6 +149,122 @@ def test_a_rejection_reaches_the_caller_when_the_retry_would_send_native_output_ assert bedrock.sent == (None,) +# Bedrock's 403 for a role without the streaming action, seen live on us.anthropic.claude-sonnet-5 on 2026-11-06 +_STREAM_PERMISSION_DENIAL: Final = BedrockError( + status_code=403, + message='{"Message":"User: arn:aws:sts::123456789012:assumed-role/invoke-only/qa is not authorized to perform: ' + "bedrock:InvokeModelWithResponseStream on resource: arn:aws:bedrock:us-west-2:123456789012:inference-profile/" + "us.anthropic.claude-sonnet-5 because no identity-based policy allows the " + 'bedrock:InvokeModelWithResponseStream action"}', +) +_INVOKE_PERMISSION_DENIAL: Final = BedrockError( + status_code=403, + message='{"Message":"User: arn:aws:sts::123456789012:assumed-role/no-bedrock/qa is not authorized to perform: ' + 'bedrock:InvokeModel because no identity-based policy allows the bedrock:InvokeModel action"}', +) +_BUFFERED_NATIVE_STREAM: Final = NativeOutputFallback( + optional_params={"outputConfig": "native", "fake_stream": True}, litellm_params={} +) + + +def _buffered(fallback: NativeOutputFallback) -> NativeOutputFallback: + return NativeOutputFallback( + optional_params={**fallback.optional_params, "fake_stream": True}, litellm_params=fallback.litellm_params + ) + + +def _call_with_buffered_stream_fallback( + bedrock: ScriptedBedrock, fallback: NativeOutputFallback | None, buffered_stream_fallback: NativeOutputFallback +) -> object: + return _resolve( + call_with_native_output_fallback( + bedrock, + None if fallback is None else lambda: fallback, + _MODEL, + buffered_stream_fallback=lambda: buffered_stream_fallback, + ) + ) + + +@_MODES +def test_a_denied_native_stream_is_sent_again_once_as_a_buffered_native_request( + asynchronous: bool, caplog: pytest.LogCaptureFixture +) -> None: + bedrock: Final = ScriptedBedrock(errors=(_STREAM_PERMISSION_DENIAL, None), asynchronous=asynchronous) + + with caplog.at_level("WARNING", logger="LiteLLM"): + response: Final = _call_with_buffered_stream_fallback(bedrock, _FALLBACK, _BUFFERED_NATIVE_STREAM) + + assert isinstance(response, ModelResponse) and response.id == "answer-2" + assert bedrock.sent == (None, _BUFFERED_NATIVE_STREAM) + assert "bedrock:InvokeModelWithResponseStream" in caplog.text and _MODEL in caplog.text + + +@_MODES +def test_a_buffered_native_request_bedrock_rejects_is_sent_again_buffered_with_the_fallback_params( + asynchronous: bool, +) -> None: + bedrock: Final = ScriptedBedrock( + errors=(_STREAM_PERMISSION_DENIAL, _GRAMMAR_REJECTION, None), asynchronous=asynchronous + ) + + response: Final = _call_with_buffered_stream_fallback(bedrock, _FALLBACK, _BUFFERED_NATIVE_STREAM) + + assert isinstance(response, ModelResponse) and response.id == "answer-3" + assert bedrock.sent == (None, _BUFFERED_NATIVE_STREAM, _buffered(_FALLBACK)) + + +@_MODES +def test_strict_tools_bedrock_rejects_after_a_buffered_native_retry_are_dropped_on_the_buffered_json_tool_retry( + asynchronous: bool, +) -> None: + bedrock: Final = ScriptedBedrock( + errors=(_STREAM_PERMISSION_DENIAL, _GRAMMAR_REJECTION, _GRAMMAR_REJECTION, None), asynchronous=asynchronous + ) + + response: Final = _resolve( + call_with_native_output_fallback( + bedrock, + lambda: _NATIVE_RETRY_WITH_STRICT_TOOLS, + "us.anthropic.claude-sonnet-4-6", + strict_tools_fallback=lambda: _STRICT_FALLBACK, + buffered_stream_fallback=lambda: _BUFFERED_NATIVE_STREAM, + ) + ) + + assert isinstance(response, ModelResponse) and response.id == "answer-4" + assert bedrock.sent == ( + None, + _BUFFERED_NATIVE_STREAM, + _buffered(_NATIVE_RETRY_WITH_STRICT_TOOLS), + _buffered(_COMPOSED_FALLBACK), + ) + + +@_MODES +@pytest.mark.parametrize( + ("error", "buffered_stream_fallback"), + [ + ( + _STREAM_PERMISSION_DENIAL, + NativeOutputFallback(optional_params={"outputConfig": "native"}, litellm_params={}), + ), + (_INVOKE_PERMISSION_DENIAL, _BUFFERED_NATIVE_STREAM), + ], + ids=["retry_would_stream_again", "invoke_denied"], +) +def test_a_permission_denial_reaches_the_caller_unless_the_retry_buffers_the_stream( + asynchronous: bool, error: BedrockError, buffered_stream_fallback: NativeOutputFallback +) -> None: + bedrock: Final = ScriptedBedrock(errors=(error,), asynchronous=asynchronous) + + with pytest.raises(BedrockError) as raised: + _call_with_buffered_stream_fallback(bedrock, _FALLBACK, buffered_stream_fallback) + + assert raised.value is error + assert bedrock.sent == (None,) + + @pytest.mark.parametrize( ("error", "advises_opting_out"), [(_REJECTION, True), (_SCHEMA_REFUSAL, False)], diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index 93bda5b444d..fdd6d5e61bc 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -21,7 +21,7 @@ from botocore.credentials import Credentials import litellm from litellm.constants import DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET -from litellm.llms.bedrock.common_utils import BedrockError +from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES, BedrockError from litellm.llms.bedrock.files.transformation import BedrockFilesConfig, BedrockJsonlFilesTransformation from tests.unit.proxy.conftest import httpx_transport @@ -2069,6 +2069,76 @@ class TestBedrockBatchAnthropicRowParams: assert ("format" in model_input.get("output_config", {})) is native_output assert (tool_names == {"json_tool_call"}) is not native_output + @pytest.mark.parametrize( + ("model", "native_over_the_limit"), + [(MODEL, True), ("bedrock/us.anthropic.claude-opus-5-5", False)], + ids=["already_native_model", "native_by_default_model"], + ) + @pytest.mark.parametrize("over_the_limit", [False, True], ids=["at_the_limit", "over_the_limit"]) + def test_only_native_by_default_models_send_json_schemas_wider_than_the_limit_through_the_json_tool( + self, local_model_cost_map, model, native_over_the_limit, over_the_limit + ): + native_output: Final = native_over_the_limit or not over_the_limit + field_names: Final = tuple( + f"field_{index:03d}" for index in range(BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES + over_the_limit) + ) + wide_schema: Final = { + "type": "object", + "properties": {name: {"type": "string"} for name in field_names}, + "required": list(field_names), + "additionalProperties": False, + } + model_input: Final = self._transform( + "/v1/chat/completions", + { + "messages": [{"role": "user", "content": "Describe a ceramic coffee mug"}], + "response_format": {"type": "json_schema", "json_schema": {"name": "wide", "schema": wide_schema}}, + }, + model=model, + ) + tool_names: Final = frozenset(tool["name"] for tool in model_input.get("tools", [])) + + assert ("format" in model_input.get("output_config", {})) is native_output + assert (tool_names == {"json_tool_call"}) is not native_output + + def test_a_deployment_opt_out_sends_uploaded_json_schema_rows_through_the_json_tool(self, local_model_cost_map): + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + record: Final = { + "custom_id": "row-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": self.MODEL, + "messages": [{"role": "user", "content": "Weather in Paris?"}], + "response_format": { + "type": "json_schema", + "json_schema": {"name": "weather", "schema": self.PARAMETERS}, + }, + }, + } + request: Final = BedrockFilesConfig().transform_create_file_request( + model=self.MODEL, + create_file_data={ + "file": ("batch.jsonl", json.dumps(record).encode(), "application/jsonl"), + "purpose": "batch", + }, + optional_params={ + "aws_access_key_id": "test-key-id", + "aws_secret_access_key": "test-secret", + "aws_region_name": "us-west-2", + }, + litellm_params={ + "s3_bucket_name": "litellm-batch-bucket", + "litellm_metadata": {"model_info": {"supports_bedrock_native_structured_output": False}}, + }, + ) + assert isinstance(request, dict) + model_input: Final = json.loads(request["data"].splitlines()[0])["modelInput"] + + assert "format" not in model_input.get("output_config", {}) + assert [tool["name"] for tool in model_input["tools"]] == ["json_tool_call"] + def test_provider_native_params_still_pass_through(self): model_input = self._transform( "/v1/chat/completions", diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index 0c1c4f5475a..7cc4629e497 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -1,11 +1,10 @@ - import asyncio, importlib, litellm, pytest import respx from httpx import Response from typing import Final -from litellm.llms.bedrock.common_utils import( +from litellm.llms.bedrock.common_utils import ( AmazonBedrockGlobalConfig, BedrockModelInfo, NativeOutputFallback, @@ -54,9 +53,7 @@ def test_bedrock_response_stream_shape_lazy_loads_once(): import litellm.llms.bedrock.common_utils as mod sentinel = MagicMock() - with patch.object( - mod, "_load_bedrock_response_stream_shape", return_value=sentinel - ) as mock_load: + with patch.object(mod, "_load_bedrock_response_stream_shape", return_value=sentinel) as mock_load: assert mod.get_bedrock_response_stream_shape() is sentinel assert mod.get_bedrock_response_stream_shape() is sentinel mock_load.assert_called_once() @@ -103,9 +100,7 @@ def test_bedrock_response_stream_shape_is_structure_shape(): from litellm.llms.bedrock.common_utils import get_bedrock_response_stream_shape loaded_shape = get_bedrock_response_stream_shape() - assert ( - loaded_shape is not None - ), "get_bedrock_response_stream_shape() is None — botocore may not be installed" + assert loaded_shape is not None, "get_bedrock_response_stream_shape() is None — botocore may not be installed" shape: StructureShape = loaded_shape assert isinstance(shape, StructureShape) assert shape.name == "ResponseStream" @@ -170,9 +165,7 @@ def test_deepseek_cris(): Test that DeepSeek models with cross-region inference prefix use converse route """ bedrock_model_info = BedrockModelInfo - bedrock_route = bedrock_model_info.get_bedrock_route( - model="bedrock/us.deepseek.r1-v1:0" - ) + bedrock_route = bedrock_model_info.get_bedrock_route(model="bedrock/us.deepseek.r1-v1:0") assert bedrock_route == "converse" @@ -228,7 +221,10 @@ def test_bare_opaque_arn_with_no_invoke_provider_routes_to_converse(arn_segment: model: Final = f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}" assert BaseAWSLLM.get_bedrock_invoke_provider(model) is None assert BedrockModelInfo.get_bedrock_route(model) == "converse" - assert BedrockModelInfo.get_bedrock_route(f"bedrock/llama/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}") == "invoke" + assert ( + BedrockModelInfo.get_bedrock_route(f"bedrock/llama/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}") + == "invoke" + ) def test_govcloud_cross_region_inference_prefix(): @@ -238,27 +234,19 @@ def test_govcloud_cross_region_inference_prefix(): bedrock_model_info = BedrockModelInfo # Test us-gov prefix is stripped correctly for Claude models - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.anthropic.claude-haiku-4-5-20251001-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-haiku-4-5-20251001-v1:0") assert base_model == "anthropic.claude-haiku-4-5-20251001-v1:0" # Test us-gov prefix is stripped correctly for different Claude versions - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0") assert base_model == "anthropic.claude-sonnet-4-5-20250929-v1:0" # Test us-gov prefix is stripped correctly for Haiku models - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.anthropic.claude-3-haiku-20240307-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-3-haiku-20240307-v1:0") assert base_model == "anthropic.claude-3-haiku-20240307-v1:0" # Test us-gov prefix is stripped correctly for Meta models - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.meta.llama3-8b-instruct-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.meta.llama3-8b-instruct-v1:0") assert base_model == "meta.llama3-8b-instruct-v1:0" @@ -272,23 +260,14 @@ def test_context_window_suffix_stripped_for_cost_lookup(): """ from litellm.llms.bedrock.common_utils import get_bedrock_base_model - assert ( - get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1[1m]") - == "anthropic.claude-opus-4-6-v1" - ) - assert ( - get_bedrock_base_model("us.anthropic.claude-sonnet-4-6[1m]") - == "anthropic.claude-sonnet-4-6" - ) + assert get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1[1m]") == "anthropic.claude-opus-4-6-v1" + assert get_bedrock_base_model("us.anthropic.claude-sonnet-4-6[1m]") == "anthropic.claude-sonnet-4-6" assert ( get_bedrock_base_model("global.anthropic.claude-opus-4-5-20251101-v1:0[1m]") == "anthropic.claude-opus-4-5-20251101-v1:0" ) # Ensure models without suffix are unaffected - assert ( - get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1") - == "anthropic.claude-opus-4-6-v1" - ) + assert get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1") == "anthropic.claude-opus-4-6-v1" # Ensure :51k throughput suffix still works assert ( get_bedrock_base_model("anthropic.claude-3-5-sonnet-20241022-v2:0:51k") @@ -358,9 +337,7 @@ def test_output_config_effort_normalization_uses_model_info_ceiling(monkeypatch) ("us.anthropic.claude-opus-4-7", "xhigh"), ], ) -def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling( - model, expected_ceiling -): +def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling(model, expected_ceiling): from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap model_info = GetModelCostMap.load_local_model_cost_map()[model] @@ -379,54 +356,24 @@ def test_route_prefix_matched_as_path_segment_not_substring(): or a ``/`` boundary. """ # The bedrock_mantle/ provider prefix must NOT be read as the mantle/ route. - assert ( - BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.5") != "mantle" - ) - assert ( - BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.4") == "converse" - ) - assert ( - BedrockModelInfo._explicit_mantle_route("bedrock_mantle/openai.gpt-5.5") - is False - ) + assert BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.5") != "mantle" + assert BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.4") == "converse" + assert BedrockModelInfo._explicit_mantle_route("bedrock_mantle/openai.gpt-5.5") is False # A genuine mantle route still resolves, via the startswith branch... - assert ( - BedrockModelInfo.get_bedrock_route("mantle/anthropic.claude-mythos-preview") - == "mantle" - ) + assert BedrockModelInfo.get_bedrock_route("mantle/anthropic.claude-mythos-preview") == "mantle" # ...and via the mid-path "/mantle/" branch (after the bedrock/ provider prefix). - assert ( - BedrockModelInfo.get_bedrock_route( - "bedrock/mantle/anthropic.claude-mythos-preview" - ) - == "mantle" - ) + assert BedrockModelInfo.get_bedrock_route("bedrock/mantle/anthropic.claude-mythos-preview") == "mantle" def test_model_has_route_prefix_exercises_both_branches(): """``_model_has_route_prefix`` matches on ``startswith`` or a ``/`` boundary only.""" # startswith branch - assert ( - BedrockModelInfo._model_has_route_prefix( - "mantle/anthropic.claude-mythos-preview", "mantle/" - ) - is True - ) + assert BedrockModelInfo._model_has_route_prefix("mantle/anthropic.claude-mythos-preview", "mantle/") is True # f"/{prefix}" boundary branch - assert ( - BedrockModelInfo._model_has_route_prefix( - "bedrock/mantle/anthropic.claude-mythos-preview", "mantle/" - ) - is True - ) + assert BedrockModelInfo._model_has_route_prefix("bedrock/mantle/anthropic.claude-mythos-preview", "mantle/") is True # neither branch: the token only appears glued to another segment - assert ( - BedrockModelInfo._model_has_route_prefix( - "bedrock_mantle/openai.gpt-5.5", "mantle/" - ) - is False - ) + assert BedrockModelInfo._model_has_route_prefix("bedrock_mantle/openai.gpt-5.5", "mantle/") is False @pytest.mark.parametrize( @@ -476,16 +423,10 @@ def test_explicit_invoke_route_does_not_match_async_invoke(): """ async_invoke_model = "async_invoke/twelvelabs.marengo-embed-2-7-v1:0" assert BedrockModelInfo._explicit_invoke_route(async_invoke_model) is False - assert ( - BedrockModelInfo._explicit_invoke_route(f"bedrock/{async_invoke_model}") - is False - ) + assert BedrockModelInfo._explicit_invoke_route(f"bedrock/{async_invoke_model}") is False # ...while async_invoke/ is still detected as its own route. assert BedrockModelInfo._explicit_async_invoke_route(async_invoke_model) is True - assert ( - BedrockModelInfo._explicit_async_invoke_route(f"bedrock/{async_invoke_model}") - is True - ) + assert BedrockModelInfo._explicit_async_invoke_route(f"bedrock/{async_invoke_model}") is True def test_capability_lookups_fall_back_to_base_model_when_regional_entry_lacks_field(monkeypatch): @@ -605,7 +546,6 @@ def test_deployment_model_info_breakpoint_flag_covers_an_unmapped_arn(local_mode assert bedrock_model_accepts_cache_points(unflagged_arn) is True - _CONVERSE_REPLY: Final = { "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, "stopReason": "end_turn", @@ -638,7 +578,11 @@ _MARKED_TOOL_CONVERSATION: Final = ( } ], }, - {"role": "tool", "tool_call_id": "call_1", "content": [{"type": "text", "text": "Sunny", "cache_control": _CACHE_MARK}]}, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [{"type": "text", "text": "Sunny", "cache_control": _CACHE_MARK}], + }, {"role": "tool", "tool_call_id": "call_1_1", "content": "Cloudy"}, {"role": "assistant", "content": "Paris is sunny, Rome is cloudy.", "cache_control": _CACHE_MARK}, {"role": "user", "content": "Thanks"}, @@ -711,6 +655,7 @@ async def test_a_bedrock_deployment_sends_cache_points_by_its_own_flag_not_a_sib "flagless": not declared, } + async def test_a_routed_bedrock_deployment_sends_cache_points_when_register_model_opted_its_model_in( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None ) -> None: @@ -738,7 +683,9 @@ async def test_a_routed_bedrock_deployment_sends_cache_points_when_register_mode num_retries=0, ) - await router.acompletion(model="flagless", messages=[{"role": "system", "content": "stable"}, {"role": "user", "content": "hi"}]) + await router.acompletion( + model="flagless", messages=[{"role": "system", "content": "stable"}, {"role": "user", "content": "hi"}] + ) assert b"cachePoint" in converse.calls.last.request.content @@ -783,10 +730,13 @@ def test_a_deployments_prompt_caching_flag_written_as_a_string_or_number_decides ) assert ( - bedrock_model_accepts_cache_points(model, model_info={"id": "qa-deployment", "supports_prompt_caching": declared}) + bedrock_model_accepts_cache_points( + model, model_info={"id": "qa-deployment", "supports_prompt_caching": declared} + ) is expected ) + def test_merge_bedrock_aws_request_params_strips_caller_identity_when_deployment_has_static_credentials(): from litellm.llms.bedrock.common_utils import merge_bedrock_aws_request_params @@ -1491,6 +1441,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="session") def event_loop(): try: @@ -1500,6 +1451,7 @@ def event_loop(): yield loop loop.close() + @pytest.fixture(scope="function") def setup_and_teardown(event_loop): import litellm @@ -1533,6 +1485,7 @@ def setup_and_teardown(event_loop): if pending: event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "set_verbose": getattr(litellm, "set_verbose", False), @@ -1547,6 +1500,7 @@ _SCALAR_DEFAULTS = { "cohere_key": getattr(litellm, "cohere_key", None), } + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestStripBedrockRoutingPrefix: """Tests for strip_bedrock_routing_prefix function.""" @@ -1577,6 +1531,7 @@ class TestStripBedrockRoutingPrefix: == "anthropic.claude-3-sonnet-20240229-v1:0" ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestStripBedrockThroughputSuffix: """Tests for strip_bedrock_throughput_suffix function.""" @@ -1604,6 +1559,7 @@ class TestStripBedrockThroughputSuffix: def test_strip_throughput_suffix(self, input_model, expected): assert strip_bedrock_throughput_suffix(input_model) == expected + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestExtractModelNameFromBedrockArn: """Tests for extract_model_name_from_bedrock_arn function.""" @@ -1624,6 +1580,7 @@ class TestExtractModelNameFromBedrockArn: arn = "ARN:aws:bedrock:us-east-1:123456789012:model/my-model" assert extract_model_name_from_bedrock_arn(arn) == "my-model" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestGetBedrockCrossRegionInferenceRegions: """Tests for get_bedrock_cross_region_inference_regions function.""" @@ -1639,6 +1596,7 @@ class TestGetBedrockCrossRegionInferenceRegions: regions = get_bedrock_cross_region_inference_regions() assert isinstance(regions, list) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestGetBedrockBaseModel: """Tests for get_bedrock_base_model function.""" @@ -1702,6 +1660,7 @@ class TestGetBedrockBaseModel: """Test that throughput tier suffixes like :51k are stripped. Issue #19113.""" assert get_bedrock_base_model(input_model) == expected + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestBedrockModelInfoWrappers: """Tests that BedrockModelInfo methods correctly wrap standalone functions.""" @@ -1723,6 +1682,7 @@ class TestBedrockModelInfoWrappers: model = "bedrock/converse/claude-3" assert BedrockModelInfo.get_non_litellm_routing_model_name(model) == strip_bedrock_routing_prefix(model) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestBedrockTokenCounter: """Tests for BedrockTokenCounter class.""" @@ -1759,12 +1719,14 @@ class TestBedrockTokenCounter: ) assert result is None + @pytest.fixture def _pr4_bedrock_env(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("AWS_ACCESS_KEY_ID", "pr4-test-aws-access-key") monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "pr4-test-aws-secret-key") monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + @pytest.fixture(scope="module") def _use_local_model_cost_map() -> Iterator[None]: with pytest.MonkeyPatch.context() as monkeypatch: @@ -1772,6 +1734,7 @@ def _use_local_model_cost_map() -> Iterator[None]: importlib.reload(litellm) yield + @pytest.mark.usefixtures( "_pr4_bedrock_env", "_use_local_model_cost_map", @@ -2348,8 +2311,12 @@ def test_bedrock_capability_model_reads_base_model_only_for_application_profile_ assert bedrock_capability_model(model, base_model) == expected -_FORMAT_REJECTION: Final = "The model returned the following errors: output_config.format: Extra inputs are not permitted" -_COMPILE_REJECTION: Final = "The model returned the following errors: compilation exceeded limits: Schema is too complex." +_FORMAT_REJECTION: Final = ( + "The model returned the following errors: output_config.format: Extra inputs are not permitted" +) +_COMPILE_REJECTION: Final = ( + "The model returned the following errors: compilation exceeded limits: Schema is too complex." +) _GRAMMAR_REJECTION: Final = ( "The model returned the following errors: The compiled grammar is too large, which would cause performance issues. " "Simplify your tool schemas or reduce the number of strict tools." @@ -2394,10 +2361,10 @@ def test_is_bedrock_native_output_rejection_matches_only_the_400s_that_reject_na ("model", "model_info", "expected"), [ ("us.anthropic.claude-opus-4-8", None, False), - ("global.anthropic.claude-sonnet-5", None, True), + ("global.anthropic.claude-sonnet-5", None, False), ("us-gov.anthropic.claude-opus-4-8", None, False), ("us.anthropic.claude-opus-4-8", {"supports_bedrock_native_structured_output": False}, False), - ("us.anthropic.claude-opus-5-5", {"supports_bedrock_native_structured_output": True}, True), + ("us.anthropic.claude-opus-5-5", {"supports_bedrock_native_structured_output": True}, False), ("us.anthropic.claude-sonnet-4-6", {"supports_bedrock_native_structured_output": True}, False), ], ids=["opus_4_8", "sonnet_5", "govcloud_opus_4_8", "deployment_opt_out", "deployment_opt_in", "native_model"], @@ -2490,7 +2457,10 @@ def test_bedrock_strict_tools_fallback_drops_only_strict_and_keeps_everything_el assert fallback == NativeOutputFallback( optional_params={ "tools": [ - {**strict_tool, "function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}}}, + { + **strict_tool, + "function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}}, + }, plain_tool, anthropic_tool, {"toolSpec": {"name": "spec", "inputSchema": {"json": {"type": "object"}}}}, diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index dd0e5762845..a758f4d4250 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -4890,6 +4890,92 @@ async def test_json_mode_stream_reaches_the_iterator_of_a_config_without_a_reque ] +class _PlainStreamRouteConfig(_TransformRecordingConfig): + def __init__(self): + super().__init__(transform_async=False) + + def get_complete_url(self, api_base, api_key, model, optional_params, litellm_params, stream=None): + return f"{api_base}/{'stream' if stream else 'answer'}" + + +class _StreamRouteConfig(_PlainStreamRouteConfig): + @property + def has_custom_stream_wrapper(self) -> bool: + return True + + +def _answers_only_the_non_streaming_route(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/answer"): + return httpx.Response(200, json=json.loads(request.content)) + return httpx.Response(403, json={"message": "not authorized to stream"}) + + +def _fake_streamed_completion( + acompletion: bool, + client: HTTPHandler | AsyncHTTPHandler, + fake_stream: object = True, + provider_config: BaseConfig | None = None, +): + return BaseLLMHTTPHandler().completion( + model="stub-model", + messages=[{"role": "user", "content": "hi"}], + api_base="https://llm.example/v1/chat", + custom_llm_provider="openai", + model_response=ModelResponse(), + encoding=None, + logging_obj=_streaming_logging_obj(), + optional_params={"fake_stream": fake_stream}, + timeout=10.0, + litellm_params={}, + acompletion=acompletion, + stream=True, + client=client, + provider_config=provider_config or _StreamRouteConfig(), + ) + + +async def test_a_fake_streamed_completion_is_sent_to_the_non_streaming_route(): + sync_client: Final = HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_answers_only_the_non_streaming_route)) + ) + async_client: Final = AsyncHTTPHandler() + async_client.client = httpx.AsyncClient(transport=httpx.MockTransport(_answers_only_the_non_streaming_route)) + + sync_chunks: Final = list(_fake_streamed_completion(acompletion=False, client=sync_client)) + async_chunks: Final = [ + chunk async for chunk in await _fake_streamed_completion(acompletion=True, client=async_client) + ] + + assert (_streamed_text(sync_chunks), _streamed_text(async_chunks)) == ("sync", "sync") + + +def _answers_only_the_streaming_route(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/stream"): + return _sse_echoing_transformed_by(request) + return httpx.Response(403, json={"message": "only streaming is allowed"}) + + +@pytest.mark.parametrize("fake_stream", ["true", 1], ids=["string", "int"]) +async def test_a_fake_stream_value_other_than_true_still_streams_from_the_streaming_route(fake_stream: object): + sync_client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_answers_only_the_streaming_route))) + async_client: Final = AsyncHTTPHandler() + async_client.client = httpx.AsyncClient(transport=httpx.MockTransport(_answers_only_the_streaming_route)) + + sync_chunks: Final = list( + _fake_streamed_completion( + acompletion=False, client=sync_client, fake_stream=fake_stream, provider_config=_PlainStreamRouteConfig() + ) + ) + async_chunks: Final = [ + chunk + async for chunk in await _fake_streamed_completion( + acompletion=True, client=async_client, fake_stream=fake_stream, provider_config=_PlainStreamRouteConfig() + ) + ] + + assert (_streamed_text(sync_chunks), _streamed_text(async_chunks)) == ("sync", "sync") + + class _ImageEditRecordingConfig(BaseImageEditConfig): def __init__(self): self.transform_calls = [] diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 6ad5ef1a4eb..79f3e697a4f 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -490,6 +490,51 @@ def test_model_info_reports_null_cost_for_unpriced_deployment_and_zero_for_decla assert input_cost > 0 and output_cost > 0 +@pytest.mark.parametrize( + ("deployment_model_info", "native"), + [ + ({}, True), + ({"supports_bedrock_native_structured_output": False}, False), + ({"supports_native_structured_output": True, "supports_bedrock_native_structured_output": False}, True), + ], + ids=["native-by-default", "bedrock-opt-out", "explicit-generic-flag"], +) +def test_model_info_reports_a_deployment_bedrock_native_output_opt_out( + local_model_cost_map: None, deployment_model_info: dict[str, object], native: bool +) -> None: + info: Final = proxy_server._enrich_model_info_with_litellm_data( + { + "model_name": "sonnet", + "litellm_params": {"model": "bedrock/us.anthropic.claude-sonnet-5"}, + "model_info": {"id": "sonnet-deployment", **deployment_model_info}, + } + )["model_info"] + + assert info["supports_native_structured_output"] is native + + +@pytest.mark.parametrize( + ("model", "by_default"), + [("bedrock/us.anthropic.claude-opus-4-7", True), ("bedrock/us.anthropic.claude-sonnet-5", False)], + ids=["set-on-a-model-the-cost-map-leaves-on-the-json-tool", "unset-on-a-model-the-cost-map-makes-native"], +) +def test_model_info_ignores_a_deployment_by_default_key_the_router_reads_only_from_the_cost_map( + local_model_cost_map: None, model: str, by_default: bool +) -> None: + def reported_native(deployment_model_info: dict[str, object]) -> object: + return proxy_server._enrich_model_info_with_litellm_data( + { + "model_name": "claude", + "litellm_params": {"model": model}, + "model_info": {"id": "claude-deployment", **deployment_model_info}, + } + )["model_info"]["supports_native_structured_output"] + + assert reported_native({"supports_bedrock_native_structured_output_by_default": by_default}) is reported_native( + {} + ) + + def test_model_info_id_lookup_reports_the_same_cost_as_the_list( client: TestClient, auth_as: Callable[[], AbstractContextManager[object]], diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index c0c8a0655fe..cd71c3f9bee 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -542,7 +542,13 @@ def test_should_not_pollute_shared_key_with_custom_nonzero_pricing(): @pytest.mark.parametrize( "flag", - ["supports_regex_lookaround", "supports_assistant_prefill", "supports_sampling_params", "supports_continuation"], + [ + "supports_regex_lookaround", + "supports_assistant_prefill", + "supports_sampling_params", + "supports_continuation", + "supports_bedrock_native_structured_output_by_default", + ], ) def test_capability_flag_stays_on_the_deployment_that_set_it(flag: str) -> None: """A deployment's capability override must not land on the shared diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index f8ec0e56e4a..b0c55d3f0f2 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1354,6 +1354,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "supports_image_size": {"type": "boolean"}, "supports_native_structured_output": {"type": "boolean"}, "supports_bedrock_native_structured_output": {"type": "boolean"}, + "supports_bedrock_native_structured_output_by_default": {"type": "boolean"}, "use_openai_responses_path": {"type": "boolean"}, "off_peak_pricing": { "type": "object", diff --git a/tests/unit/test_utils_get_model_info.py b/tests/unit/test_utils_get_model_info.py index d08742be4af..7548001ce4c 100644 --- a/tests/unit/test_utils_get_model_info.py +++ b/tests/unit/test_utils_get_model_info.py @@ -13,7 +13,7 @@ from litellm import get_model_info from litellm.llms.bedrock.common_utils import BedrockModelInfo from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.types.utils import ModelInfoBase -from litellm.utils import _invalidate_model_cost_lowercase_map +from litellm.utils import _invalidate_model_cost_lowercase_map, supports_native_structured_output import os from unittest.mock import MagicMock, patch @@ -388,6 +388,29 @@ def test_get_model_info_bedrock_cross_region_capability_parity(): assert checked > 0, "no cross-region bedrock profiles found - the filter is inert" +def test_bedrock_native_output_default_reaches_only_versions_that_read_its_key(local_model_cost_map): + flagged: Final = tuple( + key for key, row in litellm.model_cost.items() if row.get("supports_bedrock_native_structured_output_by_default") + ) + + assert flagged + assert [key for key in flagged if litellm.model_cost[key].get("supports_native_structured_output")] == [] + assert [key for key in flagged if not get_model_info(key)["supports_native_structured_output"]] == [] + + +@pytest.mark.parametrize( + ("registered", "native"), + [({"supports_bedrock_native_structured_output": False}, False), ({"max_output_tokens": 1024}, True)], + ids=["bedrock-opt-out", "unrelated-override"], +) +def test_a_registered_bedrock_opt_out_beats_the_native_output_default(local_model_cost_map, registered, native): + model: Final = "us.anthropic.claude-sonnet-5" + litellm.register_model({model: registered}, persist_across_reloads=False) + + assert get_model_info(f"bedrock/{model}")["supports_native_structured_output"] is native + assert supports_native_structured_output(f"bedrock/{model}") is native + + def _is_positive_cost(value: object) -> bool: return isinstance(value, (int, float)) and value > 0