From a94118c6cda593e9d2ea69d9b9fabdd61f11bf89 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:04:32 -0800 Subject: [PATCH] test(bedrock): expect native output on Opus 4.8 profiles after #50222 (#50343) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_bedrock_batch_files_wire.py | 14 ++++- ...t_bedrock_native_structured_output_wire.py | 56 +++++++++++++------ ...vertex_anthropic_structured_output_wire.py | 2 +- 3 files changed, 50 insertions(+), 22 deletions(-) diff --git a/tests/integration/providers/test_bedrock_batch_files_wire.py b/tests/integration/providers/test_bedrock_batch_files_wire.py index 1657895109d..162fc5aac27 100644 --- a/tests/integration/providers/test_bedrock_batch_files_wire.py +++ b/tests/integration/providers/test_bedrock_batch_files_wire.py @@ -89,8 +89,13 @@ _SCHEMA: Final = { @pytest.mark.parametrize( ("backend", "native_output"), - [("us.anthropic.claude-sonnet-5-5", False), ("us.anthropic.claude-haiku-5-5", True)], - ids=["opt_in_native_output_model", "native_output_model"], + [ + ("us.anthropic.claude-sonnet-5-5", False), + ("us.anthropic.claude-haiku-5-5", True), + ("us.anthropic.claude-opus-4-7", False), + ("us.anthropic.claude-opus-4-8", True), + ], + ids=["opt_in_native_output_model", "native_output_model", "opus_4_7", "opus_4_8"], ) def test_json_schema_batch_record_with_thinking_uploads_native_output_only_on_models_that_always_take_it( gateway: Gateway, backend: str, native_output: bool @@ -130,7 +135,10 @@ def test_json_schema_batch_record_with_thinking_uploads_native_output_only_on_mo output_format: Final = object_value(model_input.get("output_config", {})).get("format") tools: Final = tuple(object_value(entry) for entry in list_value(model_input.get("tools", []))) if native_output: - assert object_value(object_value(output_format)["schema"])["properties"] == _SCHEMA["properties"], model_input + assert model_input["output_config"] == {"format": {"type": "json_schema", "schema": _SCHEMA}}, model_input + assert model_input["anthropic_beta"] == ["structured-outputs-2025-11-13"], model_input + assert "tools" not in model_input, model_input + assert model_input["thinking"] == {"type": "adaptive"}, model_input assert not tools, model_input return assert output_format is None, model_input diff --git a/tests/integration/providers/test_bedrock_native_structured_output_wire.py b/tests/integration/providers/test_bedrock_native_structured_output_wire.py index 68942ed295b..ceb40879b9f 100644 --- a/tests/integration/providers/test_bedrock_native_structured_output_wire.py +++ b/tests/integration/providers/test_bedrock_native_structured_output_wire.py @@ -623,11 +623,9 @@ def test_messages_thinking_on_a_bedrock_converse_native_model_gets_native_output @pytest.mark.parametrize("route", ["converse", "invoke"]) -@pytest.mark.parametrize( - ("backend", "thinks_by_default"), [(_OPUS_4_8, False), (_HAIKU, True)], ids=["opus_4_8", "haiku_5_5"] -) -def test_bedrock_native_model_with_no_thinking_param_gets_native_output_only_when_it_thinks_by_default( - gateway: Gateway, route: str, backend: str, thinks_by_default: bool +@pytest.mark.parametrize("backend", [_OPUS_4_8, _HAIKU], ids=["opus_4_8", "haiku_5_5"]) +def test_bedrock_native_model_with_no_thinking_param_gets_native_output( + gateway: Gateway, route: str, backend: str ) -> None: marker, marker_hex = _marker() with wire_server(_bedrock) as wire, gateway.scenario() as scenario: @@ -640,11 +638,9 @@ def test_bedrock_native_model_with_no_thinking_param_gets_native_output_only_whe assert not message.get("tool_calls"), response.text _, body = _one_call(wire, marker_hex) invoke: Final = route == "invoke" - if thinks_by_default: - _assert_native_without_tools(body, invoke, _SCHEMA) - return - assert _native_schema(body, invoke) is None, body - assert _tools(body, invoke)[_JSON_TOOL]["properties"] == _SCHEMA["properties"], body + _assert_native_without_tools(body, invoke, _SCHEMA) + if invoke: + assert body["anthropic_beta"] == ["structured-outputs-2025-11-13"], body @pytest.mark.parametrize("route", ["converse", "invoke"]) @@ -687,8 +683,13 @@ def test_a_model_that_cannot_force_the_json_tool_retries_a_rejected_native_outpu (_NEITHER_KEY, None, False), (_OPT_IN, {"supports_bedrock_native_structured_output": True}, True), (_OPUS_4_8, {"supports_bedrock_native_structured_output": False}, False), + ( + _OPUS_4_8, + {"supports_native_structured_output": True, "supports_bedrock_native_structured_output": False}, + True, + ), ], - ids=["neither_key", "deployment_opt_in", "deployment_opt_out"], + ids=["neither_key", "deployment_opt_in", "deployment_opt_out", "explicit_native_with_bedrock_key_off"], ) def test_thinking_request_follows_the_bedrock_native_key_the_model_resolves_to( gateway: Gateway, backend: str, model_info: Mapping[str, JsonValue] | None, native: bool @@ -1047,7 +1048,7 @@ def test_bedrock_rejecting_opt_in_native_output_gets_one_retry_without_it( ids=["other_400", "native_model"], ) @pytest.mark.parametrize("route", ["converse", "invoke"]) -def test_bedrock_400_is_not_retried_unless_it_rejects_opt_in_native_output( +def test_bedrock_400_is_not_retried_unless_it_rejects_native_output( gateway: Gateway, route: str, backend: str, rejection: str ) -> None: marker, marker_hex = _marker() @@ -1058,10 +1059,19 @@ def test_bedrock_400_is_not_retried_unless_it_rejects_opt_in_native_output( "/v1/chat/completions", _chat_body(model, marker, thinking=_ADAPTIVE, response_format=_response_format(_SCHEMA)), ) - assert response.status_code == 400, response.text - assert rejection.split(": ", 1)[1] in response.text, response.text - _, body = _one_call(wire, marker_hex) - assert _native_schema(body, route == "invoke") == _SCHEMA, body + invoke: Final = route == "invoke" + if rejection == _MAX_TOKENS_400: + assert response.status_code == 400, response.text + assert rejection.split(": ", 1)[1] in response.text, response.text + _, body = _one_call(wire, marker_hex) + _assert_native_without_tools(body, invoke, _SCHEMA) + return + assert _content_json(_chat_message(response)) == _payload(marker_hex), response.text + rejected, retried = (_JSON.validate_json(request.body) for request in wire.drain()) + _assert_native_without_tools(rejected, invoke, _SCHEMA) + assert _native_schema(retried, invoke) is None, retried + assert _tools(retried, invoke)[_JSON_TOOL]["properties"] == _SCHEMA["properties"], retried + assert marker_hex in json.dumps(retried), retried def _always_rejecting(message: str) -> Callable[[Request], Reply]: @@ -1076,7 +1086,7 @@ def _always_rejecting(message: str) -> Callable[[Request], Reply]: [(_OPUS_4_8, _MAX_TOKENS_400), (_SONNET, _FABLE_REJECTION)], ids=["other_400", "native_model"], ) -def test_messages_400_on_bedrock_invoke_reaches_the_caller_after_one_call_unless_it_rejects_opt_in_native_output( +def test_messages_400_on_bedrock_invoke_reaches_the_caller_after_one_call_unless_it_rejects_native_output( gateway: Gateway, backend: str, rejection: str ) -> None: marker, marker_hex = _marker() @@ -1089,7 +1099,17 @@ def test_messages_400_on_bedrock_invoke_reaches_the_caller_after_one_call_unless ) assert response.status_code == 400, response.text assert rejection.split(": ", 1)[1] in response.text, response.text - _one_call(wire, marker_hex) + if rejection == _MAX_TOKENS_400: + _, body = _one_call(wire, marker_hex) + _assert_native_without_tools(body, True, _SCHEMA) + return + rejected, retried = (_JSON.validate_json(request.body) for request in wire.drain()) + _assert_native_without_tools(rejected, True, _SCHEMA) + assert _native_schema(retried, True) is None, retried + assert not _tools(retried, True), retried + prompt: Final = json.dumps(retried["messages"]) + assert marker_hex in prompt, retried + assert all(name in prompt for name in object_value(_SCHEMA["properties"])), retried def test_a_messages_retry_without_native_output_still_omits_the_deployments_dropped_params(gateway: Gateway) -> None: diff --git a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py index 19928ecd6e0..95e07593b2a 100644 --- a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py +++ b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py @@ -48,7 +48,7 @@ _THINKING_OFF_TOOL_MODEL: Final = "claude-opus-4-8" _UNMAPPED_MODEL: Final = "claude-scripted-9" _BEDROCK_INVOKE_MODEL: Final = "anthropic.claude-sonnet-5" _BEDROCK_TOOL_PATH_MODEL: Final = "anthropic.claude-opus-4-8" -_BEDROCK_JSON_TOOL_MODEL: Final = "us.anthropic.claude-opus-4-8" +_BEDROCK_JSON_TOOL_MODEL: Final = "us.anthropic.claude-opus-4-7" _THINKING_SIGNATURE: Final = "c2NyaXB0ZWQtc2lnbmF0dXJl" _PROJECT: Final = "scripted-project" _LOCATION: Final = "us-east5"