test(bedrock): expect native output on Opus 4.8 profiles after #50222 (#50343)

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

tests/integration/providers/test_bedrock_batch_files_wire.py

@@ -89,8 +89,13 @@ _SCHEMA: Final = {
 
 @pytest.mark.parametrize(
     ("backend", "native_output"),
-    [("us.anthropic.claude-sonnet-5-5", False), ("us.anthropic.claude-haiku-5-5", True)],
-    ids=["opt_in_native_output_model", "native_output_model"],
+    [
+        ("us.anthropic.claude-sonnet-5-5", False),
+        ("us.anthropic.claude-haiku-5-5", True),
+        ("us.anthropic.claude-opus-4-7", False),
+        ("us.anthropic.claude-opus-4-8", True),
+    ],
+    ids=["opt_in_native_output_model", "native_output_model", "opus_4_7", "opus_4_8"],
 )
 def test_json_schema_batch_record_with_thinking_uploads_native_output_only_on_models_that_always_take_it(
     gateway: Gateway, backend: str, native_output: bool
@@ -130,7 +135,10 @@ def test_json_schema_batch_record_with_thinking_uploads_native_output_only_on_mo
     output_format: Final = object_value(model_input.get("output_config", {})).get("format")
     tools: Final = tuple(object_value(entry) for entry in list_value(model_input.get("tools", [])))
     if native_output:
-        assert object_value(object_value(output_format)["schema"])["properties"] == _SCHEMA["properties"], model_input
+        assert model_input["output_config"] == {"format": {"type": "json_schema", "schema": _SCHEMA}}, model_input
+        assert model_input["anthropic_beta"] == ["structured-outputs-2025-11-13"], model_input
+        assert "tools" not in model_input, model_input
+        assert model_input["thinking"] == {"type": "adaptive"}, model_input
         assert not tools, model_input
         return
     assert output_format is None, model_input

tests/integration/providers/test_bedrock_native_structured_output_wire.py

@@ -623,11 +623,9 @@ def test_messages_thinking_on_a_bedrock_converse_native_model_gets_native_output
 
 
 @pytest.mark.parametrize("route", ["converse", "invoke"])
-@pytest.mark.parametrize(
-    ("backend", "thinks_by_default"), [(_OPUS_4_8, False), (_HAIKU, True)], ids=["opus_4_8", "haiku_5_5"]
-)
-def test_bedrock_native_model_with_no_thinking_param_gets_native_output_only_when_it_thinks_by_default(
-    gateway: Gateway, route: str, backend: str, thinks_by_default: bool
+@pytest.mark.parametrize("backend", [_OPUS_4_8, _HAIKU], ids=["opus_4_8", "haiku_5_5"])
+def test_bedrock_native_model_with_no_thinking_param_gets_native_output(
+    gateway: Gateway, route: str, backend: str
 ) -> None:
     marker, marker_hex = _marker()
     with wire_server(_bedrock) as wire, gateway.scenario() as scenario:
@@ -640,11 +638,9 @@ def test_bedrock_native_model_with_no_thinking_param_gets_native_output_only_whe
         assert not message.get("tool_calls"), response.text
         _, body = _one_call(wire, marker_hex)
         invoke: Final = route == "invoke"
-        if thinks_by_default:
-            _assert_native_without_tools(body, invoke, _SCHEMA)
-            return
-        assert _native_schema(body, invoke) is None, body
-        assert _tools(body, invoke)[_JSON_TOOL]["properties"] == _SCHEMA["properties"], body
+        _assert_native_without_tools(body, invoke, _SCHEMA)
+        if invoke:
+            assert body["anthropic_beta"] == ["structured-outputs-2025-11-13"], body
 
 
 @pytest.mark.parametrize("route", ["converse", "invoke"])
@@ -687,8 +683,13 @@ def test_a_model_that_cannot_force_the_json_tool_retries_a_rejected_native_outpu
         (_NEITHER_KEY, None, False),
         (_OPT_IN, {"supports_bedrock_native_structured_output": True}, True),
         (_OPUS_4_8, {"supports_bedrock_native_structured_output": False}, False),
+        (
+            _OPUS_4_8,
+            {"supports_native_structured_output": True, "supports_bedrock_native_structured_output": False},
+            True,
+        ),
     ],
-    ids=["neither_key", "deployment_opt_in", "deployment_opt_out"],
+    ids=["neither_key", "deployment_opt_in", "deployment_opt_out", "explicit_native_with_bedrock_key_off"],
 )
 def test_thinking_request_follows_the_bedrock_native_key_the_model_resolves_to(
     gateway: Gateway, backend: str, model_info: Mapping[str, JsonValue] | None, native: bool
@@ -1047,7 +1048,7 @@ def test_bedrock_rejecting_opt_in_native_output_gets_one_retry_without_it(
     ids=["other_400", "native_model"],
 )
 @pytest.mark.parametrize("route", ["converse", "invoke"])
-def test_bedrock_400_is_not_retried_unless_it_rejects_opt_in_native_output(
+def test_bedrock_400_is_not_retried_unless_it_rejects_native_output(
     gateway: Gateway, route: str, backend: str, rejection: str
 ) -> None:
     marker, marker_hex = _marker()
@@ -1058,10 +1059,19 @@ def test_bedrock_400_is_not_retried_unless_it_rejects_opt_in_native_output(
             "/v1/chat/completions",
             _chat_body(model, marker, thinking=_ADAPTIVE, response_format=_response_format(_SCHEMA)),
         )
-        assert response.status_code == 400, response.text
-        assert rejection.split(": ", 1)[1] in response.text, response.text
-        _, body = _one_call(wire, marker_hex)
-        assert _native_schema(body, route == "invoke") == _SCHEMA, body
+        invoke: Final = route == "invoke"
+        if rejection == _MAX_TOKENS_400:
+            assert response.status_code == 400, response.text
+            assert rejection.split(": ", 1)[1] in response.text, response.text
+            _, body = _one_call(wire, marker_hex)
+            _assert_native_without_tools(body, invoke, _SCHEMA)
+            return
+        assert _content_json(_chat_message(response)) == _payload(marker_hex), response.text
+        rejected, retried = (_JSON.validate_json(request.body) for request in wire.drain())
+        _assert_native_without_tools(rejected, invoke, _SCHEMA)
+        assert _native_schema(retried, invoke) is None, retried
+        assert _tools(retried, invoke)[_JSON_TOOL]["properties"] == _SCHEMA["properties"], retried
+        assert marker_hex in json.dumps(retried), retried
 
 
 def _always_rejecting(message: str) -> Callable[[Request], Reply]:
@@ -1076,7 +1086,7 @@ def _always_rejecting(message: str) -> Callable[[Request], Reply]:
     [(_OPUS_4_8, _MAX_TOKENS_400), (_SONNET, _FABLE_REJECTION)],
     ids=["other_400", "native_model"],
 )
-def test_messages_400_on_bedrock_invoke_reaches_the_caller_after_one_call_unless_it_rejects_opt_in_native_output(
+def test_messages_400_on_bedrock_invoke_reaches_the_caller_after_one_call_unless_it_rejects_native_output(
     gateway: Gateway, backend: str, rejection: str
 ) -> None:
     marker, marker_hex = _marker()
@@ -1089,7 +1099,17 @@ def test_messages_400_on_bedrock_invoke_reaches_the_caller_after_one_call_unless
         )
         assert response.status_code == 400, response.text
         assert rejection.split(": ", 1)[1] in response.text, response.text
-        _one_call(wire, marker_hex)
+        if rejection == _MAX_TOKENS_400:
+            _, body = _one_call(wire, marker_hex)
+            _assert_native_without_tools(body, True, _SCHEMA)
+            return
+        rejected, retried = (_JSON.validate_json(request.body) for request in wire.drain())
+        _assert_native_without_tools(rejected, True, _SCHEMA)
+        assert _native_schema(retried, True) is None, retried
+        assert not _tools(retried, True), retried
+        prompt: Final = json.dumps(retried["messages"])
+        assert marker_hex in prompt, retried
+        assert all(name in prompt for name in object_value(_SCHEMA["properties"])), retried
 
 
 def test_a_messages_retry_without_native_output_still_omits_the_deployments_dropped_params(gateway: Gateway) -> None:

tests/integration/providers/test_vertex_anthropic_structured_output_wire.py

@@ -48,7 +48,7 @@ _THINKING_OFF_TOOL_MODEL: Final = "claude-opus-4-8"
 _UNMAPPED_MODEL: Final = "claude-scripted-9"
 _BEDROCK_INVOKE_MODEL: Final = "anthropic.claude-sonnet-5"
 _BEDROCK_TOOL_PATH_MODEL: Final = "anthropic.claude-opus-4-8"
-_BEDROCK_JSON_TOOL_MODEL: Final = "us.anthropic.claude-opus-4-8"
+_BEDROCK_JSON_TOOL_MODEL: Final = "us.anthropic.claude-opus-4-7"
 _THINKING_SIGNATURE: Final = "c2NyaXB0ZWQtc2lnbmF0dXJl"
 _PROJECT: Final = "scripted-project"
 _LOCATION: Final = "us-east5"