test(e2e): cover the Bedrock provider-feature cells customers run

Adds live e2e coverage for the Bedrock combinations behind recent customer
incidents: llm_provider-* response-header forwarding on /chat/completions
(nonstream and stream), regional us.anthropic.* inference-profile ids over
the invoke route, and the Admin UI Test Connection probe for a
responses-mode Bedrock Mantle deployment. Registers the matching cells in
the coverage registry and publishes the provider x feature matrix table in
its README.

tests/e2e/coverage_registry/README.md

@@ -77,6 +77,24 @@ Strict mode exits non-zero on `@pytest.mark.covers(...)` ids that are not checke
 the registry. Add `--fail-on-collection-errors` when the job should also fail on pytest
 collection errors.
 
+## Provider x feature matrix: customer-run Bedrock combinations
+
+The provider and feature combinations customers actually run get explicit cells, expanded
+here as incidents surface new ones. The current Bedrock set, seeded from a customer's
+production shape (regional `us.anthropic.*` inference-profile ids over both chat routes,
+provider response headers for AWS-side correlation, and the Test Connection probe for a
+responses-mode Bedrock Mantle deployment):
+
+| Cell | Feature | Covering test |
+|------|---------|---------------|
+| `llm.chat_completions.bedrock_converse.basic.nonstream.works` | regional `us.` id, Converse | `llm_translation/test_chat_completions_regression_e2e.py` |
+| `llm.chat_completions.bedrock_converse.basic.stream.works` | regional `us.` id, Converse stream | `llm_translation/test_chat_completions_regression_e2e.py` |
+| `llm.chat_completions.bedrock_invoke.basic.nonstream.works` | regional `us.` id, Invoke | `llm_translation/test_bedrock_provider_matrix_e2e.py` |
+| `llm.chat_completions.bedrock_invoke.basic.stream.works` | regional `us.` id, Invoke stream | `llm_translation/test_bedrock_provider_matrix_e2e.py` |
+| `llm.chat_completions.bedrock_converse.response_headers.nonstream.works` | `llm_provider-*` headers | `llm_translation/test_bedrock_provider_matrix_e2e.py` |
+| `llm.chat_completions.bedrock_converse.response_headers.stream.works` | `llm_provider-*` headers, stream | `llm_translation/test_bedrock_provider_matrix_e2e.py` |
+| `mgmt.model.test_connection.happy_path` | Test Connection, Bedrock Mantle | `management/test_model_test_connection_e2e.py` |
+
 ## Status: this is a draft for review
 
 The cells were enumerated from the codebase and the tiers are a first proposal. Known

tests/e2e/coverage_registry/llm_conversational.yaml

@@ -29,6 +29,10 @@
 - {id: llm.chat_completions.bedrock_converse.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Bedrock vision (Anthropic/Nova)"}
 - {id: llm.chat_completions.bedrock_converse.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: bedrock_converse, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Anthropic-on-Bedrock caching"}
 - {id: llm.chat_completions.bedrock_converse.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: bedrock_converse, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Anthropic thinking on Bedrock"}
+- {id: llm.chat_completions.bedrock_converse.response_headers.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: response_headers, streaming: nonstream, assertions: [works], source: "llms/bedrock/chat/converse_handler.py:248", rationale: "Bedrock request ids must surface as llm_provider-* response headers on /chat/completions so callers can correlate calls with AWS-side logs (#37003)", fail_before_fix: proven}
+- {id: llm.chat_completions.bedrock_converse.response_headers.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: response_headers, streaming: stream, assertions: [works], source: "llms/bedrock/chat/converse_handler.py:154", rationale: "The llm_provider-* headers must also surface on streaming /chat/completions, where CustomStreamWrapper carries them instead of the nonstream setter"}
+- {id: llm.chat_completions.bedrock_invoke.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_invoke, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "Regional inference-profile ids (us.anthropic.*) over the invoke route, the deployment shape behind a customer timeout report on v1.90.0"}
+- {id: llm.chat_completions.bedrock_invoke.basic.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_invoke, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:8455", rationale: "Streaming with regional inference-profile ids over the invoke route"}
 - {id: llm.chat_completions.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Vertex AI"}
 - {id: llm.chat_completions.gemini.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini OpenAI-compatible chat translation"}
 - {id: llm.chat_completions.gemini.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini chat cost lands in SpendLogs"}

tests/e2e/coverage_registry/mgmt.yaml

@@ -75,3 +75,4 @@
 - {id: mgmt.workflow.list.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "workflow_management_endpoints.py", rationale: "Workflow tracking (smoke)"}
 - {id: mgmt.credential_migration.check.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:4252", rationale: "Encryption migration (smoke)"}
 - {id: mgmt.credential.new.serves_request, module: mgmt, tier: P1, surface: api, assertions: [serves_request], source: "credential_endpoints/endpoints.py:42", rationale: "Stored credential resolves into a deployment and serves a live /messages request"}
+- {id: mgmt.model.test_connection.happy_path, module: mgmt, tier: P0, surface: api, assertions: [happy_path], source: "_health_endpoints.py:1785", rationale: "Test Connection for a responses-mode Bedrock Mantle deployment reaches the live provider and reports success; this exact shape 500ed on an acompletion partial before v1.91.0", fail_before_fix: proven}

tests/e2e/coverage_registry/schema.py

@@ -71,6 +71,7 @@ LlmCapability = Literal[
     "pdf_input",
     "prompt_cache_1h",
     "prompt_cache_5m",
+    "response_headers",
     "service_tier",
     "structured_output",
     "thinking",

tests/e2e/llm_translation/test_bedrock_provider_matrix_e2e.py

@@ -0,0 +1,157 @@
+"""Live e2e for the Bedrock cells of the provider-feature matrix: provider
+response headers on /chat/completions and regional inference-profile model ids
+(us.anthropic.*) over the invoke route.
+
+Header forwarding is the #37003 contract: the proxy surfaces Bedrock's response
+headers prefixed llm_provider- (llm_provider-x-amzn-requestid above all) so a
+caller can hand AWS support the request id behind a completion. Regional
+inference-profile ids are the deployment shape most Bedrock customers run; a
+v1.90.0 regression timed them out, and the Converse route keeps them covered in
+test_chat_completions_regression_e2e.py, so the invoke route carries its own
+rows here.
+"""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import BaseModel
+
+from e2e_config import unique_marker
+from e2e_http import StreamingResponse, unwrap
+from lifecycle import ResourceManager
+from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody
+from passthrough_client import PassthroughClient
+
+pytestmark = pytest.mark.e2e
+
+CONVERSE_REGIONAL_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
+INVOKE_REGIONAL_BACKEND = "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0"
+PROVIDER_HEADER_PREFIX = "llm_provider-"
+BEDROCK_REQUEST_ID_HEADER = "llm_provider-x-amzn-requestid"
+
+
+class _StreamDelta(BaseModel):
+    content: str | None = None
+
+
+class _StreamChoice(BaseModel):
+    delta: _StreamDelta = _StreamDelta()
+
+
+class _StreamChunk(BaseModel):
+    choices: list[_StreamChoice] = []
+
+
+def _streamed_text(events: list[str]) -> str:
+    chunks = [_StreamChunk.model_validate_json(event) for event in events]
+    return "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices)
+
+
+def _assert_streamed_completion(result: StreamingResponse) -> None:
+    assert result.ok and result.is_streaming, f"stream was not established: {result}"
+    assert result.stream_error is None, f"stream carried an error event: {result.stream_error}"
+    assert len(result.stream_events) > 1, f"stream did not deliver multiple data events: {result}"
+    assert _streamed_text(result.stream_events).strip(), (
+        f"stream completed with no content deltas: {result.stream_events[:3]}"
+    )
+
+
+def _assert_request_id_header(result: StreamingResponse) -> None:
+    forwarded = [name for name in result.headers if name.startswith(PROVIDER_HEADER_PREFIX)]
+    assert result.headers.get(BEDROCK_REQUEST_ID_HEADER), (
+        f"missing {BEDROCK_REQUEST_ID_HEADER}; forwarded provider headers: {forwarded}"
+    )
+
+
+def _assert_completion(response: ChatResponse) -> None:
+    assert response.choices, f"completion returned no choices: {response}"
+    message = response.choices[0].message
+    content = (message.content if message else None) or ""
+    assert content.strip(), f"completion carried no content: {response}"
+
+
+def _register_bedrock_model(
+    client: PassthroughClient, resources: ResourceManager, prefix: str, backend: str
+) -> str:
+    model = f"{prefix}-{unique_marker()}"
+    model_id = client.proxy.create_model(
+        model,
+        LiteLLMParamsBody(
+            model=backend,
+            aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
+            aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
+            aws_region_name="os.environ/AWS_REGION",
+        ),
+    )
+    resources.defer(lambda: client.proxy.delete_model(model_id))
+    return model
+
+
+def _prompt() -> list[ChatMessage]:
+    return [ChatMessage(role="user", content="reply with one word")]
+
+
+class TestBedrockResponseHeaders:
+    @pytest.mark.covers(
+        "llm.chat_completions.bedrock_converse.response_headers.nonstream.works",
+        exercised_on=[],
+    )
+    def test_bedrock_request_id_header_surfaces(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        model = _register_bedrock_model(client, resources, "e2e-bedrock-headers", CONVERSE_REGIONAL_BACKEND)
+        key = resources.key()
+
+        result = client.proxy.transport.send(
+            "/chat/completions",
+            headers=client.proxy.transport.bearer(key),
+            json=ChatBody(model=model, messages=_prompt(), max_tokens=64),
+        )
+
+        assert result.ok, f"chat call failed: {result.status_code} {result.body[:300]}"
+        _assert_request_id_header(result)
+
+    @pytest.mark.covers(
+        "llm.chat_completions.bedrock_converse.response_headers.stream.works",
+        exercised_on=[],
+    )
+    def test_bedrock_request_id_header_surfaces_on_stream(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        model = _register_bedrock_model(
+            client, resources, "e2e-bedrock-headers-stream", CONVERSE_REGIONAL_BACKEND
+        )
+        key = resources.key()
+
+        result = client.proxy.chat_stream(
+            key, ChatBody(model=model, messages=_prompt(), stream=True, max_tokens=64)
+        )
+
+        _assert_streamed_completion(result)
+        _assert_request_id_header(result)
+
+
+class TestBedrockInvokeRegionalModelIds:
+    @pytest.mark.covers("llm.chat_completions.bedrock_invoke.basic.nonstream.works", exercised_on=[])
+    def test_invoke_regional_id_completes(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        model = _register_bedrock_model(client, resources, "e2e-bedrock-invoke", INVOKE_REGIONAL_BACKEND)
+        key = resources.key()
+
+        response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_prompt(), max_tokens=64)))
+
+        _assert_completion(response)
+
+    @pytest.mark.covers("llm.chat_completions.bedrock_invoke.basic.stream.works", exercised_on=[])
+    def test_invoke_regional_id_streams(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        model = _register_bedrock_model(client, resources, "e2e-bedrock-invoke-stream", INVOKE_REGIONAL_BACKEND)
+        key = resources.key()
+
+        result = client.proxy.chat_stream(
+            key, ChatBody(model=model, messages=_prompt(), stream=True, max_tokens=64)
+        )
+
+        _assert_streamed_completion(result)

tests/e2e/management/management_client.py

@@ -14,6 +14,8 @@ from e2e_http import NoBody, ProbeResult, Result, StreamingResponse, Success, Un
 from models import (
     ChatBody,
     ChatMessage,
+    ConnectionTestBody,
+    ConnectionTestResponse,
     CustomerDeleteBody,
     CustomerInfoParams,
     CustomerNewBody,
@@ -118,6 +120,17 @@ class ManagementClient:
             )
         )
 
+    def connection_test(self, body: ConnectionTestBody) -> Result[ConnectionTestResponse]:
+        """POST /health/test_connection, the call behind the Admin UI's Test
+        Connection button, probing the live provider with the supplied params."""
+        return self.proxy.transport.post(
+            "/health/test_connection",
+            headers=self.proxy.transport.master,
+            json=body,
+            response_type=ConnectionTestResponse,
+            timeout=120.0,
+        )
+
     def block_key(self, key: str) -> None:
         _ = unwrap(
             self.proxy.transport.post(

tests/e2e/management/test_model_test_connection_e2e.py

@@ -0,0 +1,42 @@
+"""Live e2e for POST /health/test_connection, the API behind the Admin UI's
+Test Connection button on the add-model form.
+
+The covered cell is a responses-mode Bedrock Mantle deployment: exactly this
+shape 500ed on a functools.partial acompletion conflict before v1.91.0 while
+every chat-mode probe stayed green, so the happy path asserts a real success
+verdict from the live provider rather than just a 200 envelope. The region is a
+literal because the endpoint rejects request-supplied os.environ/ references;
+credentials fall through to the proxy's own environment (bearer token locally,
+pod identity in CI).
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from e2e_http import unwrap
+from management_client import ManagementClient
+from models import ConnectionTestBody, LiteLLMParamsBody
+
+pytestmark = pytest.mark.e2e
+
+MANTLE_RESPONSES_BACKEND = "bedrock_mantle/openai.gpt-5.6-luna"
+MANTLE_REGION = "us-east-1"
+
+
+class TestModelTestConnection:
+    @pytest.mark.covers("mgmt.model.test_connection.happy_path")
+    def test_bedrock_mantle_responses_connection_succeeds(self, client: ManagementClient) -> None:
+        response = unwrap(
+            client.connection_test(
+                ConnectionTestBody(
+                    litellm_params=LiteLLMParamsBody(
+                        model=MANTLE_RESPONSES_BACKEND, aws_region_name=MANTLE_REGION
+                    ),
+                    mode="responses",
+                )
+            )
+        )
+
+        error = response.result.error if response.result else None
+        assert response.status == "success", f"test_connection reported an error: {error}"

tests/e2e/models.py

@@ -820,6 +820,26 @@ class ModelDeleteBody(BaseModel):
     id: str
 
 
+class ConnectionTestBody(BaseModel):
+    """POST /health/test_connection body, the API behind the Admin UI's Test
+    Connection button: the deployment params as typed into the add-model form and
+    the health-check mode picking which endpoint the probe calls. The endpoint
+    rejects `os.environ/` references, so credentials are either literal values or
+    omitted to fall through to the proxy's own environment."""
+
+    litellm_params: LiteLLMParamsBody
+    mode: Literal["chat", "completion", "embedding", "responses"]
+
+
+class ConnectionTestResult(BaseModel):
+    error: str | None = None
+
+
+class ConnectionTestResponse(BaseModel):
+    status: Literal["success", "error"]
+    result: ConnectionTestResult | None = None
+
+
 class CredentialCreateBody(BaseModel):
     credential_name: str
     credential_values: dict[str, str]