test(e2e): wait for guardrail sync in bedrock, moderation and block-code checks

All three asserted on the first call after registering a guardrail, so they were
served by a data-plane worker that had not synced it yet (~30s DB poll) and read
in-flight propagation as a guardrail that failed to block. Verified directly: the
openai_moderation guardrail lets a flagged prompt through at t=0s and returns
"Violated OpenAI moderation policy" at t=8s.

The reasoning-only responses noted in triage (content=None with reasoning_tokens
set) were a symptom of the same thing, not the cause; these are pre_call
guardrails, so a synced guardrail rejects the request before the model runs.

Add poll_until_blocked to guardrails_client for the two that surface a non-success
status, and poll on the block marker in the block_code_execution check, which
replaces the reply rather than erroring. All eight guardrail tests now pass.

Refs LIT-4821

tests/e2e/guardrails/guardrails_client.py

@@ -5,6 +5,7 @@ and chat through them on the shared ProxyClient so resources.defer cleans up.
 from __future__ import annotations
 
 import time
+from collections.abc import Callable
 from dataclasses import dataclass
 from typing import Literal
 
@@ -287,3 +288,24 @@ class GuardrailsClient:
 
 def build_client(proxy: ProxyClient) -> GuardrailsClient:
     return GuardrailsClient(proxy=proxy)
+
+
+def poll_until_blocked(call: Callable[[], Result[ChatResponse]]) -> Result[ChatResponse]:
+    """Retry a call that a guardrail should reject until it is, returning the last result.
+
+    Registering a guardrail is a control-plane write; the data-plane worker that
+    serves /chat/completions picks it up only on its next periodic DB sync (~30s in
+    proxy_server.py). A call issued right after the create therefore runs against a
+    worker that has no guardrail yet and is allowed through, which is in-flight
+    propagation rather than a guardrail that failed to block. Polling to the deadline
+    waits that out so the assertions judge the synced state; a guardrail that never
+    blocks still fails, on the last allowed result.
+    """
+    deadline = time.monotonic() + POLL_TIMEOUT
+    last = call()
+    while time.monotonic() < deadline:
+        if not isinstance(last, Success):
+            return last
+        time.sleep(POLL_INTERVAL)
+        last = call()
+    return last

tests/e2e/guardrails/test_bedrock_guardrail_e2e.py

@@ -18,7 +18,7 @@ import pytest
 
 from e2e_config import unique_marker
 from e2e_http import UnknownApiError
-from guardrails_client import GuardrailsClient
+from guardrails_client import GuardrailsClient, poll_until_blocked
 from lifecycle import ResourceManager
 
 pytestmark = pytest.mark.e2e
@@ -50,7 +50,9 @@ class TestBedrockGuardrail:
         # Selected per request rather than registered default_on, so an upstream
         # ApplyGuardrail failure surfaces here instead of 403ing every other suite
         # running against this proxy.
-        result = client.chat(scoped_key, MODEL, BLOCKED_PROMPT, guardrails=[name])
+        result = poll_until_blocked(
+            lambda: client.chat(scoped_key, MODEL, BLOCKED_PROMPT, guardrails=[name])
+        )
 
         match result:
             case UnknownApiError(status_code=status, body=body):

tests/e2e/guardrails/test_block_code_execution_guardrail_e2e.py

@@ -14,9 +14,11 @@ the shared proxy, and the chat backend is a gemini deployment created for the te
 
 from __future__ import annotations
 
+import time
+
 import pytest
 
-from e2e_config import unique_marker
+from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, unique_marker
 from e2e_http import unwrap
 from guardrails_client import BlockCodeExecutionParamsBody, GuardrailsClient
 from lifecycle import ResourceManager
@@ -54,7 +56,18 @@ class TestBlockCodeExecutionGuardrail:
         )
         resources.defer(lambda: client.delete_guardrail(guardrail_id))
 
+        # This guardrail replaces the reply rather than erroring, so wait for the
+        # block marker to appear instead of for a non-success status. The data-plane
+        # worker only picks a new guardrail up on its next DB sync (~30s), so the
+        # first call after the create is served without it.
+        deadline = time.monotonic() + POLL_TIMEOUT
         blocked = unwrap(client.chat(scoped_key, model, EXECUTION_REQUEST, guardrails=[name]))
+        while time.monotonic() < deadline:
+            if _BLOCK_MARKER in _first_content(blocked).lower():
+                break
+            time.sleep(POLL_INTERVAL)
+            blocked = unwrap(client.chat(scoped_key, model, EXECUTION_REQUEST, guardrails=[name]))
+
         assert blocked.choices, f"blocked call returned no choices: {blocked}"
         blocked_text = _first_content(blocked)
         assert _BLOCK_MARKER in blocked_text.lower(), (

tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py

@@ -16,7 +16,11 @@ import pytest
 
 from e2e_config import unique_marker
 from e2e_http import UnknownApiError, unwrap
-from guardrails_client import GuardrailsClient, OpenAIModerationParamsBody
+from guardrails_client import (
+    GuardrailsClient,
+    OpenAIModerationParamsBody,
+    poll_until_blocked,
+)
 from lifecycle import ResourceManager
 
 pytestmark = pytest.mark.e2e
@@ -45,7 +49,9 @@ class TestOpenAIModerationGuardrail:
         )
         resources.defer(lambda: client.delete_guardrail(guardrail_id))
 
-        blocked = client.chat(scoped_key, model, FLAGGED_PROMPT, guardrails=[name])
+        blocked = poll_until_blocked(
+            lambda: client.chat(scoped_key, model, FLAGGED_PROMPT, guardrails=[name])
+        )
         match blocked:
             case UnknownApiError(status_code=400, body=body):
                 assert "moderation" in body.lower(), (