From 9daff6de4c0e5de0c1c0e321c71e33ae3e35ec2c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:36:49 -0800 Subject: [PATCH] test(bedrock): guard the line's structured output path and sync stream errors The integration tests that expected an open-object schema to move to the JSON tool on a native model now expect it to stay native, as the line does. A Bedrock /v1/messages test checks a row-native profile forwards the caller's schema unchanged, and a Converse handler test checks a sync stream error keeps its status class and the line's raw body. --- ...t_bedrock_native_structured_output_wire.py | 11 ++++--- ...vertex_anthropic_structured_output_wire.py | 22 ++++---------- ...ations_anthropic_claude3_transformation.py | 18 ++++++------ .../chat/test_bedrock_converse_handler.py | 29 +++++++++++++++++++ .../test_anthropic_claude3_transformation.py | 14 +++++++-- tests/unit/llms/bedrock/test_mantle.py | 15 +++++----- 6 files changed, 68 insertions(+), 41 deletions(-) diff --git a/tests/integration/providers/test_bedrock_native_structured_output_wire.py b/tests/integration/providers/test_bedrock_native_structured_output_wire.py index 5803d1dee44..2b2fccb73c1 100644 --- a/tests/integration/providers/test_bedrock_native_structured_output_wire.py +++ b/tests/integration/providers/test_bedrock_native_structured_output_wire.py @@ -755,9 +755,7 @@ def test_native_schema_reaches_bedrock_with_limits_in_descriptions_and_every_obj @pytest.mark.parametrize("route", ["converse", "invoke"]) -def test_open_object_schema_takes_the_json_tool_on_a_native_model_and_keeps_the_dict_open( - gateway: Gateway, route: str -) -> None: +def test_open_object_schema_stays_native_on_a_model_native_on_every_request(gateway: Gateway, route: str) -> None: marker, marker_hex = _marker() with wire_server(_bedrock) as wire, gateway.scenario() as scenario: model: Final = _deployment(scenario, wire, route, _SONNET) @@ -767,9 +765,10 @@ def test_open_object_schema_takes_the_json_tool_on_a_native_model_and_keeps_the_ assert _content_json(_chat_message(response)) == _payload(marker_hex), response.text _, body = _one_call(wire, marker_hex) invoke: Final = route == "invoke" - assert _native_schema(body, invoke) is None, body - tool_schema: Final = _tools(body, invoke)[_JSON_TOOL] - assert object_value(tool_schema["properties"])["counts"] == _OPEN_COUNTS, tool_schema + schema: Final = _native_schema(body, invoke) + assert schema is not None, body + assert object_value(schema["properties"]).keys() == {"store", "counts"}, schema + assert _JSON_TOOL not in _tools(body, invoke), body def test_thinking_with_a_caller_tool_on_a_bedrock_native_model_sends_only_the_callers_tool(gateway: Gateway) -> None: diff --git a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py index bba79049283..93cb89659a2 100644 --- a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py +++ b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py @@ -1451,27 +1451,13 @@ def _native_deployment(gateway: Gateway, scenario: Scenario, api_base: str, prov raise AssertionError(provider) -@pytest.mark.parametrize("provider", ["anthropic", "azure_ai"]) -def test_native_model_json_schema_with_an_open_object_takes_the_json_tool_and_keeps_the_object_open( - gateway: Gateway, provider: str -) -> None: - marker, marker_hex = _marker("values") - with wire_server(_peer) as wire, gateway.scenario() as scenario: - model: Final = _native_deployment(gateway, scenario, wire.url, provider, _FORCED_TOOL_NATIVE_MODEL) - response: Final = gateway.request( - "POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_VALUES_RESPONSE_FORMAT) - ) - assert _chat_content(response) == _tool_input("values", marker_hex), response.text - sent: Final = _one_delivery(wire, marker_hex) - assert not _structured(sent.body), sent.body - assert _json_tool_schema(sent)["properties"] == _VALUES_SCHEMA["properties"], sent.body - - @pytest.mark.parametrize( ("provider", "backend", "extra"), [ ("anthropic", _NATIVE_MODEL, {}), ("vertex_ai", _NATIVE_MODEL, {}), + ("anthropic", _FORCED_TOOL_NATIVE_MODEL, {}), + ("azure_ai", _FORCED_TOOL_NATIVE_MODEL, {}), ("anthropic", _FORCED_TOOL_NATIVE_MODEL, {"thinking": {"type": "adaptive"}}), ("anthropic", _FORCED_TOOL_NATIVE_MODEL, {"tools": [_USER_TOOL_SPEC]}), ("anthropic", _FORCED_TOOL_NATIVE_MODEL, {"reasoning_effort": "none"}), @@ -1480,13 +1466,15 @@ def test_native_model_json_schema_with_an_open_object_takes_the_json_tool_and_ke ids=[ "forced_tool_refused-anthropic", "forced_tool_refused-vertex_ai", + "plain-anthropic", + "plain-azure_ai", "thinking", "caller_tools", "effort_none-anthropic", "effort_none-azure_ai", ], ) -def test_open_object_schema_keeps_native_output_when_the_json_tool_cannot_be_forced( +def test_open_object_schema_keeps_native_output_on_a_native_model( gateway: Gateway, provider: str, backend: str, extra: dict[str, JsonValue] ) -> None: marker, marker_hex = _marker("values") diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index 8cbaa2c7670..d4752b25bee 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -633,22 +633,22 @@ DEFINITIONS_REF_SCHEMA: Final = { def test_model_native_on_every_request_sends_every_schema_natively_without_the_json_tool( set_native_structured_output, schema, request_params ): - model = "us.anthropic.claude-opus-4-8" + model: Final = "us.anthropic.claude-opus-4-8" set_native_structured_output(model, "legacy") - result = _invoke_params_for_schema(model, schema, request_params) + result: Final = _invoke_params_for_schema(model, schema, request_params) assert result["output_format"]["type"] == "json_schema" assert "json_tool_call" not in [tool["name"] for tool in result.get("tools", [])] def test_definitions_ref_reaches_a_model_native_on_every_request_inlined(set_native_structured_output): - model = "us.anthropic.claude-opus-4-8" + model: Final = "us.anthropic.claude-opus-4-8" set_native_structured_output(model, "legacy") - result = _invoke_params_for_schema(model, DEFINITIONS_REF_SCHEMA, {}) + result: Final = _invoke_params_for_schema(model, DEFINITIONS_REF_SCHEMA, {}) - sent = result["output_format"]["schema"] + sent: Final = result["output_format"]["schema"] assert sent["properties"]["home"]["properties"]["city"] == {"type": "string"} assert "$ref" not in json.dumps(sent) @@ -657,10 +657,10 @@ def test_definitions_ref_reaches_a_model_native_on_every_request_inlined(set_nat def test_open_object_schema_stays_native_beside_the_callers_tools_on_a_model_native_on_every_request( set_native_structured_output, thinking_params ): - model = "us.anthropic.claude-opus-4-8" + model: Final = "us.anthropic.claude-opus-4-8" set_native_structured_output(model, "legacy") - result = _invoke_params_for_schema(model, OPEN_OBJECT_SCHEMA, {**thinking_params, "tools": [LOOKUP_TOOL]}) + result: Final = _invoke_params_for_schema(model, OPEN_OBJECT_SCHEMA, {**thinking_params, "tools": [LOOKUP_TOOL]}) assert "tool_choice" not in result assert [tool["name"] for tool in result["tools"]] == ["lookup"] @@ -670,10 +670,10 @@ def test_open_object_schema_stays_native_beside_the_callers_tools_on_a_model_nat def test_open_object_schema_keeps_the_json_tool_beside_the_callers_tools_on_an_opt_in_model( set_native_structured_output, ): - model = "us.anthropic.claude-opus-4-8" + model: Final = "us.anthropic.claude-opus-4-8" set_native_structured_output(model, "bedrock") - result = _invoke_params_for_schema( + result: Final = _invoke_params_for_schema( model, OPEN_OBJECT_SCHEMA, {"thinking": {"type": "adaptive"}, "tools": [LOOKUP_TOOL]} ) diff --git a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py index 407957de716..c14f37e0e36 100644 --- a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py +++ b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py @@ -346,3 +346,32 @@ def test_converse_profile_override_records_cost_without_changing_response_model( + CONVERSE_RESPONSE["usage"]["outputTokens"] * prices["output_cost_per_token"] ) + +_INVALID_REQUEST_BODY: Final = b'{"message":"The request is invalid."}' + + +@pytest.mark.parametrize( + ("status_code", "error_class"), + [(400, litellm.BadRequestError), (429, litellm.RateLimitError), (503, litellm.ServiceUnavailableError)], +) +def test_sync_converse_stream_error_keeps_the_status_class_and_raw_body( + status_code: int, error_class: type[Exception] +) -> None: + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(status_code, content=_INVALID_REQUEST_BODY) + + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + with pytest.raises(error_class) as raised: + litellm.completion( + model="bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=[{"role": "user", "content": "hi"}], + stream=True, + num_retries=0, + aws_access_key_id="test-key", + aws_secret_access_key="test-secret", + aws_region_name="us-east-1", + client=HTTPHandler(client=transport), + ) + + assert raised.value.status_code == status_code + assert repr(_INVALID_REQUEST_BODY) in raised.value.message diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index 0ea6685cec2..9cb50dd9355 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -3270,6 +3270,14 @@ MESSAGES_OPEN_OBJECT_SCHEMA: Final = { "type": "object", "properties": {"counts": {"type": "object", "additionalProperties": {"type": "integer"}}}, } +MESSAGES_KEYWORDS_SCHEMA: Final = { + "type": "object", + "properties": { + "count": {"type": "integer", "minimum": 1}, + "ids": {"type": "array", "items": {"type": "string"}, "uniqueItems": True}, + }, + "required": ["count", "ids"], +} @pytest.mark.parametrize( @@ -3288,9 +3296,11 @@ def test_bedrock_messages_schema_native_output_cannot_hold_is_inlined_on_a_bedro @pytest.mark.parametrize( - "schema", [MESSAGES_TREE_SCHEMA, MESSAGES_OPEN_OBJECT_SCHEMA], ids=["recursive", "open-object"] + "schema", + [MESSAGES_TREE_SCHEMA, MESSAGES_OPEN_OBJECT_SCHEMA, MESSAGES_KEYWORDS_SCHEMA], + ids=["recursive", "open-object", "keywords"], ) -def test_bedrock_messages_schema_native_output_cannot_hold_is_forwarded_on_a_legacy_native_profile( +def test_bedrock_messages_forwards_the_callers_schema_unchanged_on_a_legacy_native_profile( set_native_structured_output, schema ): model: Final = "us.anthropic.claude-sonnet-4-6" diff --git a/tests/unit/llms/bedrock/test_mantle.py b/tests/unit/llms/bedrock/test_mantle.py index 2b7c8a73350..cf216e81269 100644 --- a/tests/unit/llms/bedrock/test_mantle.py +++ b/tests/unit/llms/bedrock/test_mantle.py @@ -7,6 +7,7 @@ association via `aws_bedrock_project_id`. """ import json +from typing import Final from unittest.mock import patch import httpx @@ -391,25 +392,25 @@ def test_mantle_messages_validate_environment_without_project_id(): assert _MANTLE_WORKSPACE_HEADER not in headers -_MANTLE_CONFIGS = pytest.mark.parametrize( +_MANTLE_CONFIGS: Final = pytest.mark.parametrize( "config", [AmazonMantleConfig(), BedrockMantleClaudeChatConfig()], ids=["bedrock_mantle_route", "mantle_provider"] ) -_MANTLE_MODEL = "anthropic.claude-opus-4-8" -_CLOSED_RESPONSE_FORMAT = { +_MANTLE_MODEL: Final = "anthropic.claude-opus-4-8" +_CLOSED_RESPONSE_FORMAT: Final = { "type": "json_schema", "json_schema": { "name": "out", "schema": {"type": "object", "properties": {"a": {"type": "string"}}, "required": ["a"]}, }, } -_OPEN_RESPONSE_FORMAT = { +_OPEN_RESPONSE_FORMAT: Final = { "type": "json_schema", "json_schema": { "name": "out", "schema": {"type": "object", "properties": {"counts": {"type": "object", "additionalProperties": True}}}, }, } -_LOOKUP_TOOL = { +_LOOKUP_TOOL: Final = { "type": "function", "function": {"name": "lookup", "parameters": {"type": "object", "properties": {"id": {"type": "string"}}}}, } @@ -419,7 +420,7 @@ _LOOKUP_TOOL = { def test_mantle_opt_in_model_keeps_the_json_tool_when_thinking_is_requested(set_native_structured_output, config): set_native_structured_output(_MANTLE_MODEL, "bedrock") - result = config.map_openai_params( + result: Final = config.map_openai_params( non_default_params={"response_format": _CLOSED_RESPONSE_FORMAT, "thinking": {"type": "adaptive"}}, optional_params={}, model=_MANTLE_MODEL, @@ -436,7 +437,7 @@ def test_mantle_model_native_on_every_request_stays_native_beside_the_callers_to ): set_native_structured_output(_MANTLE_MODEL, "legacy") - result = config.map_openai_params( + result: Final = config.map_openai_params( non_default_params={"response_format": _OPEN_RESPONSE_FORMAT, "tools": [_LOOKUP_TOOL]}, optional_params={}, model=_MANTLE_MODEL,