From 040c8ab5bf30254804df0db23d5342aa130d7044 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:11:27 -0800 Subject: [PATCH] fix(bedrock): keep the line's structured output path where the picks broke working calls Backport line fix, not a pick. Every change keeps a request that worked on lts/1.108.x working after the picks in this branch. Models the cost map marks native on every request (Sonnet 4.6, Haiku 4.5, Sonnet 4.5, Mythos 5.1) send the caller's schema the way the line did, on Converse, Invoke and direct Anthropic. #46870 moved some of their schemas to the forced json tool. That turned array roots and typed allOf roots into 400s, gave prose for open objects with thinking on, and broke tool_choice without tools. Opt-in models (Haiku 5.5, Sonnet 5.5, Sonnet 5, Opus 4.8) keep the new native path and its schema checks. bedrock_mantle/ chat has no retry, so Mantle ignores thinking when it picks the path, and opt-in models stay on the json tool there as they did before. The sync Converse stream keeps the line's error handling, so its error classes and router retries stay the same. The retry reads the raw HTTP error there instead. It also sends the caller's headers as they were before the first attempt, so the structured output beta does not leak into it. main has the same routing gaps. --- litellm/llms/anthropic/chat/transformation.py | 22 ++-- litellm/llms/bedrock/chat/converse_handler.py | 24 ++-- .../bedrock/chat/converse_transformation.py | 6 +- .../anthropic_claude3_transformation.py | 15 ++- .../bedrock/chat/mantle/transformation.py | 8 ++ .../bedrock/chat/native_output_fallback.py | 27 +++- litellm/llms/bedrock/common_utils.py | 14 +- litellm/main.py | 3 +- .../test_bedrock_mantle_claude_chat_wire.py | 32 +++++ ...drock_converse_stream_sync_decoder_wire.py | 13 -- ...test_bedrock_native_output_fallback_sdk.py | 6 + .../test_anthropic_chat_transformation.py | 14 +- ...ations_anthropic_claude3_transformation.py | 109 ++++++++++++++-- .../chat/test_bedrock_converse_handler.py | 45 ------- .../chat/test_converse_transformation.py | 122 ++++++++++++++++-- tests/unit/llms/bedrock/test_mantle.py | 57 ++++++++ 16 files changed, 375 insertions(+), 142 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 9624dbb582b..85951566869 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1681,24 +1681,20 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): json_schema: Mapping[str, object] | None, request_params: Mapping[str, object], ) -> bool: - open_object_takes_forced_json_tool: Final = ( - json_schema is not None - and schema_has_open_object(json_schema) - and not self.is_thinking_requested(request_params) - and not self.is_thinking_enabled(dict(request_params)) - and not request_params.get("tools") - and not AnthropicModelInfo.forced_tool_use_unsupported(model) - ) - return self.uses_native_structured_output(model) and not open_object_takes_forced_json_tool + return self.uses_native_structured_output(model) - def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None: + def native_output_format_schema( + self, json_schema: Mapping[str, object], model: str | None = None + ) -> Mapping[str, object] | None: return self.filter_anthropic_output_schema(dict(self._defs_unpacked(json_schema))) - def map_response_format_to_anthropic_output_format(self, value: dict | None) -> AnthropicOutputSchema | None: + def map_response_format_to_anthropic_output_format( + self, value: dict | None, model: str | None = None + ) -> AnthropicOutputSchema | None: json_schema: Final = self._extract_json_schema_from_response_format(value) if json_schema is None: return None - native_schema: Final = self.native_output_format_schema(json_schema) + native_schema: Final = self.native_output_format_schema(json_schema, model) if native_schema is None: return None return AnthropicOutputSchema(type="json_schema", schema=dict(native_schema)) @@ -1873,7 +1869,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self._extract_json_schema_from_response_format(value), REQUEST_PARAMS.validate_python(non_default_params), ): - _output_format = self.map_response_format_to_anthropic_output_format(value) + _output_format = self.map_response_format_to_anthropic_output_format(value, model) if _output_format is not None: optional_params["output_format"] = _output_format else: diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index 3d3d05154eb..35d08d4b2c2 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -40,22 +40,14 @@ def make_sync_call( if client is None: client = get_httpx_client() # Create a new client if none provided - try: - response: Final = client.post( - api_base, - headers=headers, - data=data, - stream=not fake_stream, - logging_obj=logging_obj, - timeout=timeout, - ) - except httpx.HTTPStatusError as err: - raise BedrockError( - status_code=err.response.status_code, - message=error_response_text(err.response), - headers=err.response.headers, - response=err.response, - ) from err + response: Final = client.post( + api_base, + headers=headers, + data=data, + stream=not fake_stream, + logging_obj=logging_obj, + timeout=timeout, + ) if response.status_code != 200: raise BedrockError( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 3b88b0a3642..606975723d4 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1025,8 +1025,10 @@ class AmazonConverseConfig(BaseConfig): model, self.custom_llm_provider, thinking_requested, model_info ): return None - if BedrockModelInfo.get_base_model(model).startswith("anthropic"): - return bedrock_cleaned_output_schema(model, json_schema, self.custom_llm_provider or "bedrock_converse") + if bedrock_native_output_is_opt_in( + model, self.custom_llm_provider or "bedrock_converse", model_info + ) and BedrockModelInfo.get_base_model(model).startswith("anthropic"): + return bedrock_cleaned_output_schema(json_schema) return json_schema def _output_config_for_anthropic_format( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 3a2d5c5325d..4fb9cc15b04 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -27,6 +27,7 @@ from litellm.llms.bedrock.common_utils import ( apply_bedrock_invoke_structured_output, bedrock_chat_uses_native_structured_output, bedrock_cleaned_output_schema, + bedrock_native_output_is_opt_in, bedrock_supports_tool_search, deployment_model_info, get_anthropic_beta_from_headers, @@ -91,8 +92,12 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): def get_supported_openai_params(self, model: str) -> list[str]: return AnthropicConfig.get_supported_openai_params(self, model) - def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None: - return AnthropicConfig.native_output_schema(json_schema) + def native_output_format_schema( + self, json_schema: Mapping[str, object], model: str | None = None + ) -> Mapping[str, object] | None: + if model is not None and bedrock_native_output_is_opt_in(model, model_info=self._deployment_model_info): + return bedrock_cleaned_output_schema(json_schema) + return AnthropicConfig.native_output_format_schema(self, json_schema, model) def uses_native_structured_output_for_schema( self, @@ -102,7 +107,11 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): ) -> bool: return bedrock_chat_uses_native_structured_output( model, self.is_thinking_requested(request_params), model_info=self._deployment_model_info - ) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None) + ) and ( + json_schema is None + or not bedrock_native_output_is_opt_in(model, model_info=self._deployment_model_info) + or bedrock_cleaned_output_schema(json_schema) is not None + ) def uses_opt_in_native_output( self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object] diff --git a/litellm/llms/bedrock/chat/mantle/transformation.py b/litellm/llms/bedrock/chat/mantle/transformation.py index e48196946cd..6ef7e470110 100644 --- a/litellm/llms/bedrock/chat/mantle/transformation.py +++ b/litellm/llms/bedrock/chat/mantle/transformation.py @@ -37,6 +37,14 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig): Usage: model="bedrock/mantle/anthropic.claude-mythos-preview" """ + def uses_native_structured_output_for_schema( + self, + model: str, + json_schema: Mapping[str, object] | None, + request_params: Mapping[str, object], + ) -> bool: + return super().uses_native_structured_output_for_schema(model, json_schema, {}) + def get_complete_url( self, api_base: str | None, diff --git a/litellm/llms/bedrock/chat/native_output_fallback.py b/litellm/llms/bedrock/chat/native_output_fallback.py index 1b8ce5fcda4..55980920444 100644 --- a/litellm/llms/bedrock/chat/native_output_fallback.py +++ b/litellm/llms/bedrock/chat/native_output_fallback.py @@ -1,9 +1,12 @@ from collections.abc import Callable, Coroutine from typing import Final, TypeAlias +import httpx + from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.common_utils import ( NativeOutputFallback, + error_response_text, is_bedrock_native_output_rejection, warn_bedrock_native_output_rejected, ) @@ -16,14 +19,24 @@ NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeO NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None] -def _fallback_for( - error: BaseLLMException, fallback: NativeOutputFallbackFactory, model: str -) -> NativeOutputFallback | None: - if not is_bedrock_native_output_rejection(error.status_code, error.message): +UpstreamError: TypeAlias = BaseLLMException | httpx.HTTPStatusError + + +def _status_and_text(error: UpstreamError) -> tuple[int, str]: + match error: + case BaseLLMException(): + return error.status_code, error.message + case httpx.HTTPStatusError(): + return error.response.status_code, error_response_text(error.response) + + +def _fallback_for(error: UpstreamError, fallback: NativeOutputFallbackFactory, model: str) -> NativeOutputFallback | None: + status_code, error_text = _status_and_text(error) + if not is_bedrock_native_output_rejection(status_code, error_text): return None retry: Final = fallback() if retry is not None: - warn_bedrock_native_output_rejected(model, error.message) + warn_bedrock_native_output_rejected(model, error_text) return retry @@ -35,7 +48,7 @@ async def _await_with_native_output_fallback( ) -> NativeOutputResponse: try: return await first_attempt - except BaseLLMException as error: + except (BaseLLMException, httpx.HTTPStatusError) as error: retry: Final = _fallback_for(error, fallback, model) if retry is None: raise @@ -52,7 +65,7 @@ def call_with_native_output_fallback( return attempt(None) try: first_attempt: Final = attempt(None) - except BaseLLMException as error: + except (BaseLLMException, httpx.HTTPStatusError) as error: retry: Final = _fallback_for(error, fallback, model) if retry is None: raise diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 40f1e067d83..e7647a3f58e 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -369,15 +369,10 @@ def json_object_or_none(value: object) -> Mapping[str, object] | None: BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES: Final = 64 -def bedrock_cleaned_output_schema( - model: str, schema: Mapping[str, object], custom_llm_provider: str = "bedrock" -) -> Mapping[str, object] | None: +def bedrock_cleaned_output_schema(schema: Mapping[str, object]) -> Mapping[str, object] | None: from litellm.llms.anthropic.chat.transformation import AnthropicConfig - if ( - not _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) - and schema_property_count(schema) > BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES - ): + if schema_property_count(schema) > BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES: return None return AnthropicConfig.native_output_schema(schema) @@ -388,11 +383,10 @@ def bedrock_native_output_schema( custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None, ) -> Mapping[str, object] | None: - native_schema: Final = bedrock_cleaned_output_schema(model, schema, custom_llm_provider) if _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider): - return schema if native_schema is None else native_schema + return schema if _bedrock_native_output_opt_in(model, custom_llm_provider, model_info): - return native_schema + return bedrock_cleaned_output_schema(schema) return None diff --git a/litellm/main.py b/litellm/main.py index d03371d7ad9..42d5ecc8f20 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4296,6 +4296,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu client=client, ) else: + headers_before_first_attempt: Final = dict(headers) def invoke_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult: return base_llm_http_handler.completion( @@ -4309,7 +4310,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params), custom_llm_provider="bedrock", timeout=timeout, - headers=headers, + headers=headers if fallback is None else dict(headers_before_first_attempt), encoding=_get_encoding(), api_key=api_key, logging_obj=logging, diff --git a/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py b/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py index 2badf810b6a..2628561091b 100644 --- a/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py @@ -17,6 +17,7 @@ from litellm.constants import DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS _HAIKU: Final = "anthropic.claude-haiku-4-5" _OPUS: Final = "anthropic.claude-opus-5-5" +_OPT_IN_NATIVE_OUTPUT: Final = "anthropic.claude-opus-4-8" _SAFEGUARD: Final = "openai.gpt-oss-safeguard-120b" _API_KEY: Final = "synthetic-mantle-bearer" _ACCESS_KEY: Final = "AKIAINTEGRATION000009" @@ -504,6 +505,37 @@ def test_chat_json_schema_response_format_is_sent_as_a_forced_tool_and_returned_ assert [(request.method, request.target) for request in wire.drain()] == [("POST", _MESSAGES_PATH)] +def test_chat_json_schema_with_thinking_on_an_opt_in_model_stays_on_the_json_tool(gateway: Gateway) -> None: + marker: Final = uuid4().hex + thinking: Final[dict[str, JsonValue]] = {"type": "adaptive"} + expected: Final = _outbound( + _OPT_IN_NATIVE_OUTPUT, + [_user_turn(marker)], + tools=[{"name": "json_tool_call", "input_schema": _CITY_SCHEMA}], + tool_choice={"name": "json_tool_call", "type": "tool"}, + thinking=thinking, + ) + reply: Final = _tool_reply(marker, _OPT_IN_NATIVE_OUTPUT, "json_tool_call") + with wire_server(_bearer_peer(expected, reply)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"bedrock_mantle/{_OPT_IN_NATIVE_OUTPUT}", api_base=wire.url, api_key=_API_KEY + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": _chat_messages(marker), + "thinking": thinking, + "response_format": {"type": "json_schema", "json_schema": {"name": "weather", "schema": _CITY_SCHEMA}}, + }, + ) + assert response.status_code == 200, response.text + message: Final = object_value(_only_choice(_JSON_OBJECT.validate_json(response.content))["message"]) + assert json.loads(string_value(message["content"])) == {"city": "Paris"}, response.text + assert [(request.method, request.target) for request in wire.drain()] == [("POST", _MESSAGES_PATH)] + + def _authorization_field(part: str) -> tuple[str, str]: name, _, value = part.partition("=") return name, value diff --git a/tests/integration/sdk/test_bedrock_converse_stream_sync_decoder_wire.py b/tests/integration/sdk/test_bedrock_converse_stream_sync_decoder_wire.py index 87efbc5529e..15fa6c49216 100644 --- a/tests/integration/sdk/test_bedrock_converse_stream_sync_decoder_wire.py +++ b/tests/integration/sdk/test_bedrock_converse_stream_sync_decoder_wire.py @@ -99,16 +99,3 @@ def test_k03_sync_stream_with_normal_frames_delivers_the_text_and_a_stop() -> No assert consumed.finish_reasons[-1] == "stop", consumed assert len(wire.drain()) == 1 - -def test_k04_sync_stream_open_400_carries_the_bedrock_body_as_text() -> None: - body: Final = '{"message":"The request is invalid."}' - - def reject(_: Request) -> Reply: - return Reply(status=400, body=body.encode()) - - with wire_server(reject) as wire: - with pytest.raises(BadRequestError) as raised: - _consume_sync_stream(wire) - assert raised.value.status_code == 400, raised.value - assert f"BedrockException - {body}" in raised.value.message, raised.value.message - assert len(wire.drain()) == 1 diff --git a/tests/integration/sdk/test_bedrock_native_output_fallback_sdk.py b/tests/integration/sdk/test_bedrock_native_output_fallback_sdk.py index 745c1ceb603..9b53a51fef9 100644 --- a/tests/integration/sdk/test_bedrock_native_output_fallback_sdk.py +++ b/tests/integration/sdk/test_bedrock_native_output_fallback_sdk.py @@ -211,12 +211,18 @@ def _sent_bodies( "output_config": {"effort": "low"}, "response_format": _response_format(_SCHEMA), }, + { + "thinking": {"type": "adaptive"}, + "response_format": _response_format(_SCHEMA), + "extra_headers": {"anthropic-beta": "context-1m-2025-08-07"}, + }, ], ids=[ "reasoning_effort", "constrained_schema", "response_format_and_output_config_format", "adaptive_thinking_with_effort", + "caller_beta_header", ], ) def test_the_retry_sends_what_a_deployment_without_native_output_sends( diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index e4ed52ee1da..36e6200315f 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1346,15 +1346,15 @@ def test_opus_uses_native_structured_output(model_name): @pytest.mark.parametrize( - ("field_schema", "native"), + "field_schema", [ - ({"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, True), - ({"type": "object", "additionalProperties": {"type": "integer"}}, False), - ({"anyOf": [{"type": "object"}, {"type": "null"}]}, False), + {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + {"type": "object", "additionalProperties": {"type": "integer"}}, + {"anyOf": [{"type": "object"}, {"type": "null"}]}, ], ids=["closed-object", "dict-field", "nullable-dict-field"], ) -def test_dict_fields_take_the_json_tool_on_a_native_structured_output_model(field_schema, native): +def test_every_field_shape_keeps_native_output_on_a_native_structured_output_model(field_schema): optional_params: Final = AnthropicConfig().map_openai_params( non_default_params={ "response_format": { @@ -1370,8 +1370,8 @@ def test_dict_fields_take_the_json_tool_on_a_native_structured_output_model(fiel drop_params=False, ) - assert ("output_format" in optional_params) is native - assert ([tool["name"] for tool in optional_params.get("tools", [])] == ["json_tool_call"]) is not native + assert "output_format" in optional_params + assert "tools" not in optional_params diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index a23bc88afc5..8cbaa2c7670 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -582,13 +582,10 @@ def test_bedrock_native_output_key_applies_only_when_thinking_is_requested( assert ("tools" in result) is not native -@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) @pytest.mark.parametrize("schema", [TREE_SCHEMA, OPEN_OBJECT_SCHEMA], ids=["recursive", "open-object"]) -def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_a_native_profile( - set_native_structured_output, schema, native_key -): +def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_an_opt_in_model(set_native_structured_output, schema): model = "us.anthropic.claude-opus-4-8" - set_native_structured_output(model, native_key) + set_native_structured_output(model, "bedrock") result = _invoke_params_for_schema(model, schema) @@ -596,6 +593,94 @@ def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_a_native_profil assert [tool["name"] for tool in result["tools"]] == ["json_tool_call"] +LOOKUP_TOOL: Final = { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object", "properties": {"id": {"type": "string"}}}}, +} + +ARRAY_ROOT_SCHEMA: Final = {"type": "array", "items": {"type": "object", "additionalProperties": {"type": "string"}}} +TYPED_ALL_OF_ROOT_SCHEMA: Final = { + "type": "object", + "allOf": [ + {"type": "object", "properties": {"a": {"type": "string"}}, "required": ["a"]}, + {"type": "object", "properties": {"b": {"type": "string"}}, "required": ["b"]}, + ], +} +DEFINITIONS_REF_SCHEMA: Final = { + "type": "object", + "properties": {"home": {"$ref": "#/definitions/Address"}}, + "required": ["home"], + "definitions": {"Address": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}, +} + + +@pytest.mark.parametrize( + "schema", + [ARRAY_ROOT_SCHEMA, TYPED_ALL_OF_ROOT_SCHEMA, OPEN_OBJECT_SCHEMA, TREE_SCHEMA, DEFINITIONS_REF_SCHEMA], + ids=["array-root", "typed-all-of-root", "open-object", "recursive", "definitions-ref"], +) +@pytest.mark.parametrize( + "request_params", + [ + {}, + {"thinking": {"type": "adaptive"}}, + {"reasoning_effort": "low"}, + {"tool_choice": "none"}, + {"tool_choice": {"type": "function", "function": {"name": "lookup"}}}, + ], + ids=["plain", "thinking", "reasoning-effort", "tool-choice-none", "named-tool-choice-without-tools"], +) +def test_model_native_on_every_request_sends_every_schema_natively_without_the_json_tool( + set_native_structured_output, schema, request_params +): + model = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, "legacy") + + result = _invoke_params_for_schema(model, schema, request_params) + + assert result["output_format"]["type"] == "json_schema" + assert "json_tool_call" not in [tool["name"] for tool in result.get("tools", [])] + + +def test_definitions_ref_reaches_a_model_native_on_every_request_inlined(set_native_structured_output): + model = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, "legacy") + + result = _invoke_params_for_schema(model, DEFINITIONS_REF_SCHEMA, {}) + + sent = result["output_format"]["schema"] + assert sent["properties"]["home"]["properties"]["city"] == {"type": "string"} + assert "$ref" not in json.dumps(sent) + + +@pytest.mark.parametrize("thinking_params", [{}, {"thinking": {"type": "adaptive"}}], ids=["no-thinking", "thinking"]) +def test_open_object_schema_stays_native_beside_the_callers_tools_on_a_model_native_on_every_request( + set_native_structured_output, thinking_params +): + model = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, "legacy") + + result = _invoke_params_for_schema(model, OPEN_OBJECT_SCHEMA, {**thinking_params, "tools": [LOOKUP_TOOL]}) + + assert "tool_choice" not in result + assert [tool["name"] for tool in result["tools"]] == ["lookup"] + assert result["output_format"]["schema"]["properties"]["counts"]["additionalProperties"] is False + + +def test_open_object_schema_keeps_the_json_tool_beside_the_callers_tools_on_an_opt_in_model( + set_native_structured_output, +): + model = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, "bedrock") + + result = _invoke_params_for_schema( + model, OPEN_OBJECT_SCHEMA, {"thinking": {"type": "adaptive"}, "tools": [LOOKUP_TOOL]} + ) + + assert "output_format" not in result + assert sorted(tool["name"] for tool in result["tools"]) == ["json_tool_call", "lookup"] + + @pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) def test_closed_schema_goes_native_on_either_native_key(set_native_structured_output, native_key): model = "us.anthropic.claude-opus-4-8" @@ -609,10 +694,11 @@ def test_closed_schema_goes_native_on_either_native_key(set_native_structured_ou assert "tools" not in result -@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) -def test_shared_defs_reach_native_output_once_instead_of_inlined_per_use(set_native_structured_output, native_key): +def test_shared_defs_reach_native_output_once_instead_of_inlined_per_use_on_an_opt_in_model( + set_native_structured_output, +): model = "us.anthropic.claude-opus-4-8" - set_native_structured_output(model, native_key) + set_native_structured_output(model, "bedrock") party_ref = {"$ref": "#/$defs/Party"} schema = { "type": "object", @@ -649,12 +735,11 @@ class PuzzleAnswer(BaseModel): relabels: list[Relabel] -@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) -def test_nested_pydantic_response_format_reaches_native_output_with_refs_that_resolve( - set_native_structured_output, native_key +def test_nested_pydantic_response_format_reaches_native_output_with_refs_that_resolve_on_an_opt_in_model( + set_native_structured_output, ): model = "us.anthropic.claude-haiku-5-5" - set_native_structured_output(model, native_key) + set_native_structured_output(model, "bedrock") config: Final = AmazonAnthropicClaudeConfig() result: Final = config.map_openai_params( diff --git a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py index ad14ae7c8a2..407957de716 100644 --- a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py +++ b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py @@ -346,48 +346,3 @@ def test_converse_profile_override_records_cost_without_changing_response_model( + CONVERSE_RESPONSE["usage"]["outputTokens"] * prices["output_cost_per_token"] ) - -_UPSTREAM_ERROR_BODY: Final = '{"message":"The request is invalid."}' - - -@pytest.mark.usefixtures("local_model_cost_map") -@pytest.mark.parametrize( - "model", ["us.anthropic.claude-sonnet-4-5-20250929-v1:0", "us.anthropic.claude-opus-4-8"], ids=["sonnet", "opus"] -) -@pytest.mark.parametrize( - ("upstream_status", "error_class", "status_code"), - [ - (400, litellm.BadRequestError, 400), - (429, litellm.RateLimitError, 429), - (500, litellm.ServiceUnavailableError, 503), - (503, litellm.ServiceUnavailableError, 503), - ], -) -def test_sync_converse_streaming_upstream_error_carries_the_upstream_body_as_text( - model: str, - upstream_status: int, - error_class: type[litellm.BadRequestError | litellm.RateLimitError | litellm.ServiceUnavailableError], - status_code: int, -) -> None: - client: Final = HTTPHandler( - client=httpx.Client( - transport=httpx.MockTransport( - lambda request: httpx.Response(upstream_status, content=_UPSTREAM_ERROR_BODY.encode()) - ) - ) - ) - - with pytest.raises(error_class) as raised: - litellm.completion( - model=f"bedrock/converse/{model}", - messages=[{"role": "user", "content": "hi"}], - stream=True, - num_retries=0, - aws_access_key_id="fake", - aws_secret_access_key="fake", - aws_region_name="us-east-1", - client=client, - ) - - assert raised.value.status_code == status_code - assert raised.value.message == f"litellm.{error_class.__name__}: BedrockException - {_UPSTREAM_ERROR_BODY}" diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 04d7c9a672c..49d99f7123e 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -5081,12 +5081,15 @@ def _schema_keys(node: object) -> frozenset[str]: return frozenset() -def _converse_request_for_schema(model: str, schema: Mapping[str, object]) -> RequestObject: +def _converse_request_for_schema( + model: str, schema: Mapping[str, object], tools: tuple[Mapping[str, object], ...] = () +) -> RequestObject: config: Final = AmazonConverseConfig() optional_params: Final = config.map_openai_params( non_default_params={ "response_format": {"type": "json_schema", "json_schema": {"name": "out", "schema": schema}}, "thinking": ADAPTIVE_THINKING, + **({"tools": list(tools)} if tools else {}), }, optional_params={}, model=model, @@ -5101,12 +5104,11 @@ def _converse_request_for_schema(model: str, schema: Mapping[str, object]) -> Re ) -@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) -def test_native_output_schema_drops_keywords_bedrock_rejects_and_closes_objects( - set_native_structured_output, native_key +def test_native_output_schema_drops_keywords_bedrock_rejects_and_closes_objects_on_an_opt_in_model( + set_native_structured_output, ): model: Final = "us.anthropic.claude-opus-4-8" - set_native_structured_output(model, native_key) + set_native_structured_output(model, "bedrock") request: Final = _converse_request_for_schema(model, CONSTRAINED_PERSON_SCHEMA) @@ -5120,7 +5122,6 @@ def test_native_output_schema_drops_keywords_bedrock_rejects_and_closes_objects( assert sent["properties"]["address"]["additionalProperties"] is False -@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) @pytest.mark.parametrize( ("schema", "kept_keyword"), [ @@ -5132,11 +5133,11 @@ def test_native_output_schema_drops_keywords_bedrock_rejects_and_closes_objects( ], ids=["recursive", "root-ref", "open-object", "lookaround-pattern", "unlisted-format"], ) -def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_a_native_profile( - set_native_structured_output, schema, kept_keyword, native_key +def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_an_opt_in_model( + set_native_structured_output, schema, kept_keyword ): model: Final = "us.anthropic.claude-opus-4-8" - set_native_structured_output(model, native_key) + set_native_structured_output(model, "bedrock") request: Final = _converse_request_for_schema(model, schema) @@ -5146,6 +5147,102 @@ def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_a_native_profil assert kept_keyword in _schema_keys(tool_spec["inputSchema"]["json"]) +LOOKUP_TOOL: Final = { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object", "properties": {"id": {"type": "string"}}}}, +} + +ARRAY_ROOT_SCHEMA: Final = {"type": "array", "items": {"type": "object", "additionalProperties": {"type": "string"}}} +TYPED_ALL_OF_ROOT_SCHEMA: Final = { + "type": "object", + "allOf": [ + {"type": "object", "properties": {"a": {"type": "string"}}, "required": ["a"]}, + {"type": "object", "properties": {"b": {"type": "string"}}, "required": ["b"]}, + ], +} + + +@pytest.mark.parametrize( + "schema", + [ + ARRAY_ROOT_SCHEMA, + TYPED_ALL_OF_ROOT_SCHEMA, + OPEN_OBJECT_SCHEMA, + CONSTRAINED_PERSON_SCHEMA, + LOOKAROUND_PATTERN_SCHEMA, + RECURSIVE_TREE_SCHEMA, + ], + ids=["array-root", "typed-all-of-root", "open-object", "constrained", "lookaround-pattern", "recursive"], +) +@pytest.mark.parametrize( + "request_params", + [ + {}, + {"thinking": ADAPTIVE_THINKING}, + {"reasoning_effort": "low"}, + {"tool_choice": "auto"}, + {"tool_choice": {"type": "function", "function": {"name": "lookup"}}}, + ], + ids=["plain", "thinking", "reasoning-effort", "tool-choice-auto", "named-tool-choice-without-tools"], +) +def test_model_native_on_every_request_sends_the_callers_schema_natively_with_every_keyword( + set_native_structured_output, schema, request_params +): + model: Final = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, "legacy") + config: Final = AmazonConverseConfig() + optional_params: Final = config.map_openai_params( + non_default_params={ + "response_format": {"type": "json_schema", "json_schema": {"name": "out", "schema": schema}}, + **request_params, + }, + optional_params={}, + model=model, + drop_params=False, + ) + + request: Final = config._transform_request( + model=model, + messages=[{"role": "user", "content": "Fill in a plausible example."}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + sent: Final = json.loads(request["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"]) + assert _schema_keys(schema) <= _schema_keys(sent) + assert "json_tool_call" not in [tool["toolSpec"]["name"] for tool in request.get("toolConfig", {}).get("tools", [])] + + +def _tool_names(request: RequestObject) -> list[str]: + return sorted(tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]) + + +def test_open_object_schema_stays_native_beside_the_callers_tools_on_a_model_native_on_every_request( + set_native_structured_output, +): + model: Final = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, "legacy") + + request: Final = _converse_request_for_schema(model, OPEN_OBJECT_SCHEMA, (LOOKUP_TOOL,)) + + assert "outputConfig" in request + assert _tool_names(request) == ["lookup"] + assert "toolChoice" not in request["toolConfig"] + + +def test_open_object_schema_keeps_the_json_tool_beside_the_callers_tools_on_an_opt_in_model( + set_native_structured_output, +): + model: Final = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, "bedrock") + + request: Final = _converse_request_for_schema(model, OPEN_OBJECT_SCHEMA, (LOOKUP_TOOL,)) + + assert "outputConfig" not in request + assert _tool_names(request) == ["json_tool_call", "lookup"] + + def _shared_defs_schema(defs_key: str) -> Mapping[str, object]: party_ref: Final = {"$ref": f"#/{defs_key}/Party"} return { @@ -5195,7 +5292,6 @@ def _optional_groups_schema(count: int, group_size: int = 6) -> Mapping[str, obj return {"type": "object", "properties": groups, "required": list(groups)} -@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) @pytest.mark.parametrize( ("schema_for", "limit"), [ @@ -5212,11 +5308,11 @@ def _optional_groups_schema(count: int, group_size: int = 6) -> Mapping[str, obj ], ids=["type-list-unions", "any-of-unions", "optional", "optional-grammar-branches"], ) -def test_schema_past_a_native_output_size_limit_keeps_the_tool_path( - set_native_structured_output, native_key, schema_for, limit +def test_schema_past_a_native_output_size_limit_keeps_the_tool_path_on_an_opt_in_model( + set_native_structured_output, schema_for, limit ): model: Final = "us.anthropic.claude-opus-4-8" - set_native_structured_output(model, native_key) + set_native_structured_output(model, "bedrock") at_limit: Final = _converse_request_for_schema(model, schema_for(limit)) past_limit: Final = _converse_request_for_schema(model, schema_for(limit + 1)) diff --git a/tests/unit/llms/bedrock/test_mantle.py b/tests/unit/llms/bedrock/test_mantle.py index 63af0105f5b..2b7c8a73350 100644 --- a/tests/unit/llms/bedrock/test_mantle.py +++ b/tests/unit/llms/bedrock/test_mantle.py @@ -17,6 +17,7 @@ from litellm.llms.bedrock.chat.mantle.transformation import AmazonMantleConfig from litellm.llms.bedrock.messages.mantle_transformation import ( AmazonMantleMessagesConfig, ) +from litellm.llms.bedrock_mantle.chat.claude_transformation import BedrockMantleClaudeChatConfig # AWS names this header for Mantle workspaces on the Anthropic Messages API, checked 2026-10-02: # https://docs.aws.amazon.com/bedrock/latest/userguide/workspaces.html @@ -390,6 +391,62 @@ def test_mantle_messages_validate_environment_without_project_id(): assert _MANTLE_WORKSPACE_HEADER not in headers +_MANTLE_CONFIGS = pytest.mark.parametrize( + "config", [AmazonMantleConfig(), BedrockMantleClaudeChatConfig()], ids=["bedrock_mantle_route", "mantle_provider"] +) +_MANTLE_MODEL = "anthropic.claude-opus-4-8" +_CLOSED_RESPONSE_FORMAT = { + "type": "json_schema", + "json_schema": { + "name": "out", + "schema": {"type": "object", "properties": {"a": {"type": "string"}}, "required": ["a"]}, + }, +} +_OPEN_RESPONSE_FORMAT = { + "type": "json_schema", + "json_schema": { + "name": "out", + "schema": {"type": "object", "properties": {"counts": {"type": "object", "additionalProperties": True}}}, + }, +} +_LOOKUP_TOOL = { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object", "properties": {"id": {"type": "string"}}}}, +} + + +@_MANTLE_CONFIGS +def test_mantle_opt_in_model_keeps_the_json_tool_when_thinking_is_requested(set_native_structured_output, config): + set_native_structured_output(_MANTLE_MODEL, "bedrock") + + result = config.map_openai_params( + non_default_params={"response_format": _CLOSED_RESPONSE_FORMAT, "thinking": {"type": "adaptive"}}, + optional_params={}, + model=_MANTLE_MODEL, + drop_params=False, + ) + + assert "output_format" not in result + assert [tool["name"] for tool in result["tools"]] == ["json_tool_call"] + + +@_MANTLE_CONFIGS +def test_mantle_model_native_on_every_request_stays_native_beside_the_callers_tools( + set_native_structured_output, config +): + set_native_structured_output(_MANTLE_MODEL, "legacy") + + result = config.map_openai_params( + non_default_params={"response_format": _OPEN_RESPONSE_FORMAT, "tools": [_LOOKUP_TOOL]}, + optional_params={}, + model=_MANTLE_MODEL, + drop_params=False, + ) + + assert "output_format" in result + assert [tool["name"] for tool in result["tools"]] == ["lookup"] + + def test_mantle_completion_sends_workspace_header_and_clean_body(): import litellm