diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 9624dbb582b..85951566869 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1681,24 +1681,20 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): json_schema: Mapping[str, object] | None, request_params: Mapping[str, object], ) -> bool: - open_object_takes_forced_json_tool: Final = ( - json_schema is not None - and schema_has_open_object(json_schema) - and not self.is_thinking_requested(request_params) - and not self.is_thinking_enabled(dict(request_params)) - and not request_params.get("tools") - and not AnthropicModelInfo.forced_tool_use_unsupported(model) - ) - return self.uses_native_structured_output(model) and not open_object_takes_forced_json_tool + return self.uses_native_structured_output(model) - def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None: + def native_output_format_schema( + self, json_schema: Mapping[str, object], model: str | None = None + ) -> Mapping[str, object] | None: return self.filter_anthropic_output_schema(dict(self._defs_unpacked(json_schema))) - def map_response_format_to_anthropic_output_format(self, value: dict | None) -> AnthropicOutputSchema | None: + def map_response_format_to_anthropic_output_format( + self, value: dict | None, model: str | None = None + ) -> AnthropicOutputSchema | None: json_schema: Final = self._extract_json_schema_from_response_format(value) if json_schema is None: return None - native_schema: Final = self.native_output_format_schema(json_schema) + native_schema: Final = self.native_output_format_schema(json_schema, model) if native_schema is None: return None return AnthropicOutputSchema(type="json_schema", schema=dict(native_schema)) @@ -1873,7 +1869,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self._extract_json_schema_from_response_format(value), REQUEST_PARAMS.validate_python(non_default_params), ): - _output_format = self.map_response_format_to_anthropic_output_format(value) + _output_format = self.map_response_format_to_anthropic_output_format(value, model) if _output_format is not None: optional_params["output_format"] = _output_format else: diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index 3d3d05154eb..35d08d4b2c2 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -40,22 +40,14 @@ def make_sync_call( if client is None: client = get_httpx_client() # Create a new client if none provided - try: - response: Final = client.post( - api_base, - headers=headers, - data=data, - stream=not fake_stream, - logging_obj=logging_obj, - timeout=timeout, - ) - except httpx.HTTPStatusError as err: - raise BedrockError( - status_code=err.response.status_code, - message=error_response_text(err.response), - headers=err.response.headers, - response=err.response, - ) from err + response: Final = client.post( + api_base, + headers=headers, + data=data, + stream=not fake_stream, + logging_obj=logging_obj, + timeout=timeout, + ) if response.status_code != 200: raise BedrockError( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 3b88b0a3642..606975723d4 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1025,8 +1025,10 @@ class AmazonConverseConfig(BaseConfig): model, self.custom_llm_provider, thinking_requested, model_info ): return None - if BedrockModelInfo.get_base_model(model).startswith("anthropic"): - return bedrock_cleaned_output_schema(model, json_schema, self.custom_llm_provider or "bedrock_converse") + if bedrock_native_output_is_opt_in( + model, self.custom_llm_provider or "bedrock_converse", model_info + ) and BedrockModelInfo.get_base_model(model).startswith("anthropic"): + return bedrock_cleaned_output_schema(json_schema) return json_schema def _output_config_for_anthropic_format( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 3a2d5c5325d..4fb9cc15b04 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -27,6 +27,7 @@ from litellm.llms.bedrock.common_utils import ( apply_bedrock_invoke_structured_output, bedrock_chat_uses_native_structured_output, bedrock_cleaned_output_schema, + bedrock_native_output_is_opt_in, bedrock_supports_tool_search, deployment_model_info, get_anthropic_beta_from_headers, @@ -91,8 +92,12 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): def get_supported_openai_params(self, model: str) -> list[str]: return AnthropicConfig.get_supported_openai_params(self, model) - def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None: - return AnthropicConfig.native_output_schema(json_schema) + def native_output_format_schema( + self, json_schema: Mapping[str, object], model: str | None = None + ) -> Mapping[str, object] | None: + if model is not None and bedrock_native_output_is_opt_in(model, model_info=self._deployment_model_info): + return bedrock_cleaned_output_schema(json_schema) + return AnthropicConfig.native_output_format_schema(self, json_schema, model) def uses_native_structured_output_for_schema( self, @@ -102,7 +107,11 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): ) -> bool: return bedrock_chat_uses_native_structured_output( model, self.is_thinking_requested(request_params), model_info=self._deployment_model_info - ) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None) + ) and ( + json_schema is None + or not bedrock_native_output_is_opt_in(model, model_info=self._deployment_model_info) + or bedrock_cleaned_output_schema(json_schema) is not None + ) def uses_opt_in_native_output( self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object] diff --git a/litellm/llms/bedrock/chat/mantle/transformation.py b/litellm/llms/bedrock/chat/mantle/transformation.py index e48196946cd..6ef7e470110 100644 --- a/litellm/llms/bedrock/chat/mantle/transformation.py +++ b/litellm/llms/bedrock/chat/mantle/transformation.py @@ -37,6 +37,14 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig): Usage: model="bedrock/mantle/anthropic.claude-mythos-preview" """ + def uses_native_structured_output_for_schema( + self, + model: str, + json_schema: Mapping[str, object] | None, + request_params: Mapping[str, object], + ) -> bool: + return super().uses_native_structured_output_for_schema(model, json_schema, {}) + def get_complete_url( self, api_base: str | None, diff --git a/litellm/llms/bedrock/chat/native_output_fallback.py b/litellm/llms/bedrock/chat/native_output_fallback.py index 1b8ce5fcda4..55980920444 100644 --- a/litellm/llms/bedrock/chat/native_output_fallback.py +++ b/litellm/llms/bedrock/chat/native_output_fallback.py @@ -1,9 +1,12 @@ from collections.abc import Callable, Coroutine from typing import Final, TypeAlias +import httpx + from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.common_utils import ( NativeOutputFallback, + error_response_text, is_bedrock_native_output_rejection, warn_bedrock_native_output_rejected, ) @@ -16,14 +19,24 @@ NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeO NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None] -def _fallback_for( - error: BaseLLMException, fallback: NativeOutputFallbackFactory, model: str -) -> NativeOutputFallback | None: - if not is_bedrock_native_output_rejection(error.status_code, error.message): +UpstreamError: TypeAlias = BaseLLMException | httpx.HTTPStatusError + + +def _status_and_text(error: UpstreamError) -> tuple[int, str]: + match error: + case BaseLLMException(): + return error.status_code, error.message + case httpx.HTTPStatusError(): + return error.response.status_code, error_response_text(error.response) + + +def _fallback_for(error: UpstreamError, fallback: NativeOutputFallbackFactory, model: str) -> NativeOutputFallback | None: + status_code, error_text = _status_and_text(error) + if not is_bedrock_native_output_rejection(status_code, error_text): return None retry: Final = fallback() if retry is not None: - warn_bedrock_native_output_rejected(model, error.message) + warn_bedrock_native_output_rejected(model, error_text) return retry @@ -35,7 +48,7 @@ async def _await_with_native_output_fallback( ) -> NativeOutputResponse: try: return await first_attempt - except BaseLLMException as error: + except (BaseLLMException, httpx.HTTPStatusError) as error: retry: Final = _fallback_for(error, fallback, model) if retry is None: raise @@ -52,7 +65,7 @@ def call_with_native_output_fallback( return attempt(None) try: first_attempt: Final = attempt(None) - except BaseLLMException as error: + except (BaseLLMException, httpx.HTTPStatusError) as error: retry: Final = _fallback_for(error, fallback, model) if retry is None: raise diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 40f1e067d83..e7647a3f58e 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -369,15 +369,10 @@ def json_object_or_none(value: object) -> Mapping[str, object] | None: BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES: Final = 64 -def bedrock_cleaned_output_schema( - model: str, schema: Mapping[str, object], custom_llm_provider: str = "bedrock" -) -> Mapping[str, object] | None: +def bedrock_cleaned_output_schema(schema: Mapping[str, object]) -> Mapping[str, object] | None: from litellm.llms.anthropic.chat.transformation import AnthropicConfig - if ( - not _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) - and schema_property_count(schema) > BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES - ): + if schema_property_count(schema) > BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES: return None return AnthropicConfig.native_output_schema(schema) @@ -388,11 +383,10 @@ def bedrock_native_output_schema( custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None, ) -> Mapping[str, object] | None: - native_schema: Final = bedrock_cleaned_output_schema(model, schema, custom_llm_provider) if _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider): - return schema if native_schema is None else native_schema + return schema if _bedrock_native_output_opt_in(model, custom_llm_provider, model_info): - return native_schema + return bedrock_cleaned_output_schema(schema) return None diff --git a/litellm/main.py b/litellm/main.py index d03371d7ad9..42d5ecc8f20 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4296,6 +4296,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu client=client, ) else: + headers_before_first_attempt: Final = dict(headers) def invoke_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult: return base_llm_http_handler.completion( @@ -4309,7 +4310,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params), custom_llm_provider="bedrock", timeout=timeout, - headers=headers, + headers=headers if fallback is None else dict(headers_before_first_attempt), encoding=_get_encoding(), api_key=api_key, logging_obj=logging, diff --git a/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py b/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py index 2badf810b6a..2628561091b 100644 --- a/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py @@ -17,6 +17,7 @@ from litellm.constants import DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS _HAIKU: Final = "anthropic.claude-haiku-4-5" _OPUS: Final = "anthropic.claude-opus-5-5" +_OPT_IN_NATIVE_OUTPUT: Final = "anthropic.claude-opus-4-8" _SAFEGUARD: Final = "openai.gpt-oss-safeguard-120b" _API_KEY: Final = "synthetic-mantle-bearer" _ACCESS_KEY: Final = "AKIAINTEGRATION000009" @@ -504,6 +505,37 @@ def test_chat_json_schema_response_format_is_sent_as_a_forced_tool_and_returned_ assert [(request.method, request.target) for request in wire.drain()] == [("POST", _MESSAGES_PATH)] +def test_chat_json_schema_with_thinking_on_an_opt_in_model_stays_on_the_json_tool(gateway: Gateway) -> None: + marker: Final = uuid4().hex + thinking: Final[dict[str, JsonValue]] = {"type": "adaptive"} + expected: Final = _outbound( + _OPT_IN_NATIVE_OUTPUT, + [_user_turn(marker)], + tools=[{"name": "json_tool_call", "input_schema": _CITY_SCHEMA}], + tool_choice={"name": "json_tool_call", "type": "tool"}, + thinking=thinking, + ) + reply: Final = _tool_reply(marker, _OPT_IN_NATIVE_OUTPUT, "json_tool_call") + with wire_server(_bearer_peer(expected, reply)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"bedrock_mantle/{_OPT_IN_NATIVE_OUTPUT}", api_base=wire.url, api_key=_API_KEY + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": _chat_messages(marker), + "thinking": thinking, + "response_format": {"type": "json_schema", "json_schema": {"name": "weather", "schema": _CITY_SCHEMA}}, + }, + ) + assert response.status_code == 200, response.text + message: Final = object_value(_only_choice(_JSON_OBJECT.validate_json(response.content))["message"]) + assert json.loads(string_value(message["content"])) == {"city": "Paris"}, response.text + assert [(request.method, request.target) for request in wire.drain()] == [("POST", _MESSAGES_PATH)] + + def _authorization_field(part: str) -> tuple[str, str]: name, _, value = part.partition("=") return name, value diff --git a/tests/integration/sdk/test_bedrock_converse_stream_sync_decoder_wire.py b/tests/integration/sdk/test_bedrock_converse_stream_sync_decoder_wire.py index 87efbc5529e..15fa6c49216 100644 --- a/tests/integration/sdk/test_bedrock_converse_stream_sync_decoder_wire.py +++ b/tests/integration/sdk/test_bedrock_converse_stream_sync_decoder_wire.py @@ -99,16 +99,3 @@ def test_k03_sync_stream_with_normal_frames_delivers_the_text_and_a_stop() -> No assert consumed.finish_reasons[-1] == "stop", consumed assert len(wire.drain()) == 1 - -def test_k04_sync_stream_open_400_carries_the_bedrock_body_as_text() -> None: - body: Final = '{"message":"The request is invalid."}' - - def reject(_: Request) -> Reply: - return Reply(status=400, body=body.encode()) - - with wire_server(reject) as wire: - with pytest.raises(BadRequestError) as raised: - _consume_sync_stream(wire) - assert raised.value.status_code == 400, raised.value - assert f"BedrockException - {body}" in raised.value.message, raised.value.message - assert len(wire.drain()) == 1 diff --git a/tests/integration/sdk/test_bedrock_native_output_fallback_sdk.py b/tests/integration/sdk/test_bedrock_native_output_fallback_sdk.py index 745c1ceb603..9b53a51fef9 100644 --- a/tests/integration/sdk/test_bedrock_native_output_fallback_sdk.py +++ b/tests/integration/sdk/test_bedrock_native_output_fallback_sdk.py @@ -211,12 +211,18 @@ def _sent_bodies( "output_config": {"effort": "low"}, "response_format": _response_format(_SCHEMA), }, + { + "thinking": {"type": "adaptive"}, + "response_format": _response_format(_SCHEMA), + "extra_headers": {"anthropic-beta": "context-1m-2025-08-07"}, + }, ], ids=[ "reasoning_effort", "constrained_schema", "response_format_and_output_config_format", "adaptive_thinking_with_effort", + "caller_beta_header", ], ) def test_the_retry_sends_what_a_deployment_without_native_output_sends( diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index e4ed52ee1da..36e6200315f 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1346,15 +1346,15 @@ def test_opus_uses_native_structured_output(model_name): @pytest.mark.parametrize( - ("field_schema", "native"), + "field_schema", [ - ({"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, True), - ({"type": "object", "additionalProperties": {"type": "integer"}}, False), - ({"anyOf": [{"type": "object"}, {"type": "null"}]}, False), + {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + {"type": "object", "additionalProperties": {"type": "integer"}}, + {"anyOf": [{"type": "object"}, {"type": "null"}]}, ], ids=["closed-object", "dict-field", "nullable-dict-field"], ) -def test_dict_fields_take_the_json_tool_on_a_native_structured_output_model(field_schema, native): +def test_every_field_shape_keeps_native_output_on_a_native_structured_output_model(field_schema): optional_params: Final = AnthropicConfig().map_openai_params( non_default_params={ "response_format": { @@ -1370,8 +1370,8 @@ def test_dict_fields_take_the_json_tool_on_a_native_structured_output_model(fiel drop_params=False, ) - assert ("output_format" in optional_params) is native - assert ([tool["name"] for tool in optional_params.get("tools", [])] == ["json_tool_call"]) is not native + assert "output_format" in optional_params + assert "tools" not in optional_params diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index a23bc88afc5..8cbaa2c7670 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -582,13 +582,10 @@ def test_bedrock_native_output_key_applies_only_when_thinking_is_requested( assert ("tools" in result) is not native -@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) @pytest.mark.parametrize("schema", [TREE_SCHEMA, OPEN_OBJECT_SCHEMA], ids=["recursive", "open-object"]) -def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_a_native_profile( - set_native_structured_output, schema, native_key -): +def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_an_opt_in_model(set_native_structured_output, schema): model = "us.anthropic.claude-opus-4-8" - set_native_structured_output(model, native_key) + set_native_structured_output(model, "bedrock") result = _invoke_params_for_schema(model, schema) @@ -596,6 +593,94 @@ def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_a_native_profil assert [tool["name"] for tool in result["tools"]] == ["json_tool_call"] +LOOKUP_TOOL: Final = { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object", "properties": {"id": {"type": "string"}}}}, +} + +ARRAY_ROOT_SCHEMA: Final = {"type": "array", "items": {"type": "object", "additionalProperties": {"type": "string"}}} +TYPED_ALL_OF_ROOT_SCHEMA: Final = { + "type": "object", + "allOf": [ + {"type": "object", "properties": {"a": {"type": "string"}}, "required": ["a"]}, + {"type": "object", "properties": {"b": {"type": "string"}}, "required": ["b"]}, + ], +} +DEFINITIONS_REF_SCHEMA: Final = { + "type": "object", + "properties": {"home": {"$ref": "#/definitions/Address"}}, + "required": ["home"], + "definitions": {"Address": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}, +} + + +@pytest.mark.parametrize( + "schema", + [ARRAY_ROOT_SCHEMA, TYPED_ALL_OF_ROOT_SCHEMA, OPEN_OBJECT_SCHEMA, TREE_SCHEMA, DEFINITIONS_REF_SCHEMA], + ids=["array-root", "typed-all-of-root", "open-object", "recursive", "definitions-ref"], +) +@pytest.mark.parametrize( + "request_params", + [ + {}, + {"thinking": {"type": "adaptive"}}, + {"reasoning_effort": "low"}, + {"tool_choice": "none"}, + {"tool_choice": {"type": "function", "function": {"name": "lookup"}}}, + ], + ids=["plain", "thinking", "reasoning-effort", "tool-choice-none", "named-tool-choice-without-tools"], +) +def test_model_native_on_every_request_sends_every_schema_natively_without_the_json_tool( + set_native_structured_output, schema, request_params +): + model = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, "legacy") + + result = _invoke_params_for_schema(model, schema, request_params) + + assert result["output_format"]["type"] == "json_schema" + assert "json_tool_call" not in [tool["name"] for tool in result.get("tools", [])] + + +def test_definitions_ref_reaches_a_model_native_on_every_request_inlined(set_native_structured_output): + model = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, "legacy") + + result = _invoke_params_for_schema(model, DEFINITIONS_REF_SCHEMA, {}) + + sent = result["output_format"]["schema"] + assert sent["properties"]["home"]["properties"]["city"] == {"type": "string"} + assert "$ref" not in json.dumps(sent) + + +@pytest.mark.parametrize("thinking_params", [{}, {"thinking": {"type": "adaptive"}}], ids=["no-thinking", "thinking"]) +def test_open_object_schema_stays_native_beside_the_callers_tools_on_a_model_native_on_every_request( + set_native_structured_output, thinking_params +): + model = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, "legacy") + + result = _invoke_params_for_schema(model, OPEN_OBJECT_SCHEMA, {**thinking_params, "tools": [LOOKUP_TOOL]}) + + assert "tool_choice" not in result + assert [tool["name"] for tool in result["tools"]] == ["lookup"] + assert result["output_format"]["schema"]["properties"]["counts"]["additionalProperties"] is False + + +def test_open_object_schema_keeps_the_json_tool_beside_the_callers_tools_on_an_opt_in_model( + set_native_structured_output, +): + model = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, "bedrock") + + result = _invoke_params_for_schema( + model, OPEN_OBJECT_SCHEMA, {"thinking": {"type": "adaptive"}, "tools": [LOOKUP_TOOL]} + ) + + assert "output_format" not in result + assert sorted(tool["name"] for tool in result["tools"]) == ["json_tool_call", "lookup"] + + @pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) def test_closed_schema_goes_native_on_either_native_key(set_native_structured_output, native_key): model = "us.anthropic.claude-opus-4-8" @@ -609,10 +694,11 @@ def test_closed_schema_goes_native_on_either_native_key(set_native_structured_ou assert "tools" not in result -@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) -def test_shared_defs_reach_native_output_once_instead_of_inlined_per_use(set_native_structured_output, native_key): +def test_shared_defs_reach_native_output_once_instead_of_inlined_per_use_on_an_opt_in_model( + set_native_structured_output, +): model = "us.anthropic.claude-opus-4-8" - set_native_structured_output(model, native_key) + set_native_structured_output(model, "bedrock") party_ref = {"$ref": "#/$defs/Party"} schema = { "type": "object", @@ -649,12 +735,11 @@ class PuzzleAnswer(BaseModel): relabels: list[Relabel] -@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) -def test_nested_pydantic_response_format_reaches_native_output_with_refs_that_resolve( - set_native_structured_output, native_key +def test_nested_pydantic_response_format_reaches_native_output_with_refs_that_resolve_on_an_opt_in_model( + set_native_structured_output, ): model = "us.anthropic.claude-haiku-5-5" - set_native_structured_output(model, native_key) + set_native_structured_output(model, "bedrock") config: Final = AmazonAnthropicClaudeConfig() result: Final = config.map_openai_params( diff --git a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py index ad14ae7c8a2..407957de716 100644 --- a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py +++ b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py @@ -346,48 +346,3 @@ def test_converse_profile_override_records_cost_without_changing_response_model( + CONVERSE_RESPONSE["usage"]["outputTokens"] * prices["output_cost_per_token"] ) - -_UPSTREAM_ERROR_BODY: Final = '{"message":"The request is invalid."}' - - -@pytest.mark.usefixtures("local_model_cost_map") -@pytest.mark.parametrize( - "model", ["us.anthropic.claude-sonnet-4-5-20250929-v1:0", "us.anthropic.claude-opus-4-8"], ids=["sonnet", "opus"] -) -@pytest.mark.parametrize( - ("upstream_status", "error_class", "status_code"), - [ - (400, litellm.BadRequestError, 400), - (429, litellm.RateLimitError, 429), - (500, litellm.ServiceUnavailableError, 503), - (503, litellm.ServiceUnavailableError, 503), - ], -) -def test_sync_converse_streaming_upstream_error_carries_the_upstream_body_as_text( - model: str, - upstream_status: int, - error_class: type[litellm.BadRequestError | litellm.RateLimitError | litellm.ServiceUnavailableError], - status_code: int, -) -> None: - client: Final = HTTPHandler( - client=httpx.Client( - transport=httpx.MockTransport( - lambda request: httpx.Response(upstream_status, content=_UPSTREAM_ERROR_BODY.encode()) - ) - ) - ) - - with pytest.raises(error_class) as raised: - litellm.completion( - model=f"bedrock/converse/{model}", - messages=[{"role": "user", "content": "hi"}], - stream=True, - num_retries=0, - aws_access_key_id="fake", - aws_secret_access_key="fake", - aws_region_name="us-east-1", - client=client, - ) - - assert raised.value.status_code == status_code - assert raised.value.message == f"litellm.{error_class.__name__}: BedrockException - {_UPSTREAM_ERROR_BODY}" diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 04d7c9a672c..49d99f7123e 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -5081,12 +5081,15 @@ def _schema_keys(node: object) -> frozenset[str]: return frozenset() -def _converse_request_for_schema(model: str, schema: Mapping[str, object]) -> RequestObject: +def _converse_request_for_schema( + model: str, schema: Mapping[str, object], tools: tuple[Mapping[str, object], ...] = () +) -> RequestObject: config: Final = AmazonConverseConfig() optional_params: Final = config.map_openai_params( non_default_params={ "response_format": {"type": "json_schema", "json_schema": {"name": "out", "schema": schema}}, "thinking": ADAPTIVE_THINKING, + **({"tools": list(tools)} if tools else {}), }, optional_params={}, model=model, @@ -5101,12 +5104,11 @@ def _converse_request_for_schema(model: str, schema: Mapping[str, object]) -> Re ) -@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) -def test_native_output_schema_drops_keywords_bedrock_rejects_and_closes_objects( - set_native_structured_output, native_key +def test_native_output_schema_drops_keywords_bedrock_rejects_and_closes_objects_on_an_opt_in_model( + set_native_structured_output, ): model: Final = "us.anthropic.claude-opus-4-8" - set_native_structured_output(model, native_key) + set_native_structured_output(model, "bedrock") request: Final = _converse_request_for_schema(model, CONSTRAINED_PERSON_SCHEMA) @@ -5120,7 +5122,6 @@ def test_native_output_schema_drops_keywords_bedrock_rejects_and_closes_objects( assert sent["properties"]["address"]["additionalProperties"] is False -@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) @pytest.mark.parametrize( ("schema", "kept_keyword"), [ @@ -5132,11 +5133,11 @@ def test_native_output_schema_drops_keywords_bedrock_rejects_and_closes_objects( ], ids=["recursive", "root-ref", "open-object", "lookaround-pattern", "unlisted-format"], ) -def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_a_native_profile( - set_native_structured_output, schema, kept_keyword, native_key +def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_an_opt_in_model( + set_native_structured_output, schema, kept_keyword ): model: Final = "us.anthropic.claude-opus-4-8" - set_native_structured_output(model, native_key) + set_native_structured_output(model, "bedrock") request: Final = _converse_request_for_schema(model, schema) @@ -5146,6 +5147,102 @@ def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_a_native_profil assert kept_keyword in _schema_keys(tool_spec["inputSchema"]["json"]) +LOOKUP_TOOL: Final = { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object", "properties": {"id": {"type": "string"}}}}, +} + +ARRAY_ROOT_SCHEMA: Final = {"type": "array", "items": {"type": "object", "additionalProperties": {"type": "string"}}} +TYPED_ALL_OF_ROOT_SCHEMA: Final = { + "type": "object", + "allOf": [ + {"type": "object", "properties": {"a": {"type": "string"}}, "required": ["a"]}, + {"type": "object", "properties": {"b": {"type": "string"}}, "required": ["b"]}, + ], +} + + +@pytest.mark.parametrize( + "schema", + [ + ARRAY_ROOT_SCHEMA, + TYPED_ALL_OF_ROOT_SCHEMA, + OPEN_OBJECT_SCHEMA, + CONSTRAINED_PERSON_SCHEMA, + LOOKAROUND_PATTERN_SCHEMA, + RECURSIVE_TREE_SCHEMA, + ], + ids=["array-root", "typed-all-of-root", "open-object", "constrained", "lookaround-pattern", "recursive"], +) +@pytest.mark.parametrize( + "request_params", + [ + {}, + {"thinking": ADAPTIVE_THINKING}, + {"reasoning_effort": "low"}, + {"tool_choice": "auto"}, + {"tool_choice": {"type": "function", "function": {"name": "lookup"}}}, + ], + ids=["plain", "thinking", "reasoning-effort", "tool-choice-auto", "named-tool-choice-without-tools"], +) +def test_model_native_on_every_request_sends_the_callers_schema_natively_with_every_keyword( + set_native_structured_output, schema, request_params +): + model: Final = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, "legacy") + config: Final = AmazonConverseConfig() + optional_params: Final = config.map_openai_params( + non_default_params={ + "response_format": {"type": "json_schema", "json_schema": {"name": "out", "schema": schema}}, + **request_params, + }, + optional_params={}, + model=model, + drop_params=False, + ) + + request: Final = config._transform_request( + model=model, + messages=[{"role": "user", "content": "Fill in a plausible example."}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + sent: Final = json.loads(request["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"]) + assert _schema_keys(schema) <= _schema_keys(sent) + assert "json_tool_call" not in [tool["toolSpec"]["name"] for tool in request.get("toolConfig", {}).get("tools", [])] + + +def _tool_names(request: RequestObject) -> list[str]: + return sorted(tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]) + + +def test_open_object_schema_stays_native_beside_the_callers_tools_on_a_model_native_on_every_request( + set_native_structured_output, +): + model: Final = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, "legacy") + + request: Final = _converse_request_for_schema(model, OPEN_OBJECT_SCHEMA, (LOOKUP_TOOL,)) + + assert "outputConfig" in request + assert _tool_names(request) == ["lookup"] + assert "toolChoice" not in request["toolConfig"] + + +def test_open_object_schema_keeps_the_json_tool_beside_the_callers_tools_on_an_opt_in_model( + set_native_structured_output, +): + model: Final = "us.anthropic.claude-opus-4-8" + set_native_structured_output(model, "bedrock") + + request: Final = _converse_request_for_schema(model, OPEN_OBJECT_SCHEMA, (LOOKUP_TOOL,)) + + assert "outputConfig" not in request + assert _tool_names(request) == ["json_tool_call", "lookup"] + + def _shared_defs_schema(defs_key: str) -> Mapping[str, object]: party_ref: Final = {"$ref": f"#/{defs_key}/Party"} return { @@ -5195,7 +5292,6 @@ def _optional_groups_schema(count: int, group_size: int = 6) -> Mapping[str, obj return {"type": "object", "properties": groups, "required": list(groups)} -@pytest.mark.parametrize("native_key", ["legacy", "bedrock"]) @pytest.mark.parametrize( ("schema_for", "limit"), [ @@ -5212,11 +5308,11 @@ def _optional_groups_schema(count: int, group_size: int = 6) -> Mapping[str, obj ], ids=["type-list-unions", "any-of-unions", "optional", "optional-grammar-branches"], ) -def test_schema_past_a_native_output_size_limit_keeps_the_tool_path( - set_native_structured_output, native_key, schema_for, limit +def test_schema_past_a_native_output_size_limit_keeps_the_tool_path_on_an_opt_in_model( + set_native_structured_output, schema_for, limit ): model: Final = "us.anthropic.claude-opus-4-8" - set_native_structured_output(model, native_key) + set_native_structured_output(model, "bedrock") at_limit: Final = _converse_request_for_schema(model, schema_for(limit)) past_limit: Final = _converse_request_for_schema(model, schema_for(limit + 1)) diff --git a/tests/unit/llms/bedrock/test_mantle.py b/tests/unit/llms/bedrock/test_mantle.py index 63af0105f5b..2b7c8a73350 100644 --- a/tests/unit/llms/bedrock/test_mantle.py +++ b/tests/unit/llms/bedrock/test_mantle.py @@ -17,6 +17,7 @@ from litellm.llms.bedrock.chat.mantle.transformation import AmazonMantleConfig from litellm.llms.bedrock.messages.mantle_transformation import ( AmazonMantleMessagesConfig, ) +from litellm.llms.bedrock_mantle.chat.claude_transformation import BedrockMantleClaudeChatConfig # AWS names this header for Mantle workspaces on the Anthropic Messages API, checked 2026-10-02: # https://docs.aws.amazon.com/bedrock/latest/userguide/workspaces.html @@ -390,6 +391,62 @@ def test_mantle_messages_validate_environment_without_project_id(): assert _MANTLE_WORKSPACE_HEADER not in headers +_MANTLE_CONFIGS = pytest.mark.parametrize( + "config", [AmazonMantleConfig(), BedrockMantleClaudeChatConfig()], ids=["bedrock_mantle_route", "mantle_provider"] +) +_MANTLE_MODEL = "anthropic.claude-opus-4-8" +_CLOSED_RESPONSE_FORMAT = { + "type": "json_schema", + "json_schema": { + "name": "out", + "schema": {"type": "object", "properties": {"a": {"type": "string"}}, "required": ["a"]}, + }, +} +_OPEN_RESPONSE_FORMAT = { + "type": "json_schema", + "json_schema": { + "name": "out", + "schema": {"type": "object", "properties": {"counts": {"type": "object", "additionalProperties": True}}}, + }, +} +_LOOKUP_TOOL = { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object", "properties": {"id": {"type": "string"}}}}, +} + + +@_MANTLE_CONFIGS +def test_mantle_opt_in_model_keeps_the_json_tool_when_thinking_is_requested(set_native_structured_output, config): + set_native_structured_output(_MANTLE_MODEL, "bedrock") + + result = config.map_openai_params( + non_default_params={"response_format": _CLOSED_RESPONSE_FORMAT, "thinking": {"type": "adaptive"}}, + optional_params={}, + model=_MANTLE_MODEL, + drop_params=False, + ) + + assert "output_format" not in result + assert [tool["name"] for tool in result["tools"]] == ["json_tool_call"] + + +@_MANTLE_CONFIGS +def test_mantle_model_native_on_every_request_stays_native_beside_the_callers_tools( + set_native_structured_output, config +): + set_native_structured_output(_MANTLE_MODEL, "legacy") + + result = config.map_openai_params( + non_default_params={"response_format": _OPEN_RESPONSE_FORMAT, "tools": [_LOOKUP_TOOL]}, + optional_params={}, + model=_MANTLE_MODEL, + drop_params=False, + ) + + assert "output_format" in result + assert [tool["name"] for tool in result["tools"]] == ["lookup"] + + def test_mantle_completion_sends_workspace_header_and_clean_body(): import litellm