From f744899784f44c9c6405427faf5780c2e31ef004 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Mon, 27 Jul 2026 13:32:35 -0700 Subject: [PATCH] test(e2e): wait for guardrail sync in bedrock, moderation and block-code checks All three asserted on the first call after registering a guardrail, so they were served by a data-plane worker that had not synced it yet (~30s DB poll) and read in-flight propagation as a guardrail that failed to block. Verified directly: the openai_moderation guardrail lets a flagged prompt through at t=0s and returns "Violated OpenAI moderation policy" at t=8s. The reasoning-only responses noted in triage (content=None with reasoning_tokens set) were a symptom of the same thing, not the cause; these are pre_call guardrails, so a synced guardrail rejects the request before the model runs. Add poll_until_blocked to guardrails_client for the two that surface a non-success status, and poll on the block marker in the block_code_execution check, which replaces the reply rather than erroring. All eight guardrail tests now pass. Refs LIT-4821 --- tests/e2e/guardrails/guardrails_client.py | 22 +++++++++++++++++++ .../guardrails/test_bedrock_guardrail_e2e.py | 6 +++-- ...test_block_code_execution_guardrail_e2e.py | 15 ++++++++++++- .../test_openai_moderation_guardrail_e2e.py | 10 +++++++-- 4 files changed, 48 insertions(+), 5 deletions(-) diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py index d56e4e9311a..5a54a4f0bbc 100644 --- a/tests/e2e/guardrails/guardrails_client.py +++ b/tests/e2e/guardrails/guardrails_client.py @@ -5,6 +5,7 @@ and chat through them on the shared ProxyClient so resources.defer cleans up. from __future__ import annotations import time +from collections.abc import Callable from dataclasses import dataclass from typing import Literal @@ -287,3 +288,24 @@ class GuardrailsClient: def build_client(proxy: ProxyClient) -> GuardrailsClient: return GuardrailsClient(proxy=proxy) + + +def poll_until_blocked(call: Callable[[], Result[ChatResponse]]) -> Result[ChatResponse]: + """Retry a call that a guardrail should reject until it is, returning the last result. + + Registering a guardrail is a control-plane write; the data-plane worker that + serves /chat/completions picks it up only on its next periodic DB sync (~30s in + proxy_server.py). A call issued right after the create therefore runs against a + worker that has no guardrail yet and is allowed through, which is in-flight + propagation rather than a guardrail that failed to block. Polling to the deadline + waits that out so the assertions judge the synced state; a guardrail that never + blocks still fails, on the last allowed result. + """ + deadline = time.monotonic() + POLL_TIMEOUT + last = call() + while time.monotonic() < deadline: + if not isinstance(last, Success): + return last + time.sleep(POLL_INTERVAL) + last = call() + return last diff --git a/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py b/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py index ba3c5071cbb..dd61e630d7d 100644 --- a/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py @@ -18,7 +18,7 @@ import pytest from e2e_config import unique_marker from e2e_http import UnknownApiError -from guardrails_client import GuardrailsClient +from guardrails_client import GuardrailsClient, poll_until_blocked from lifecycle import ResourceManager pytestmark = pytest.mark.e2e @@ -50,7 +50,9 @@ class TestBedrockGuardrail: # Selected per request rather than registered default_on, so an upstream # ApplyGuardrail failure surfaces here instead of 403ing every other suite # running against this proxy. - result = client.chat(scoped_key, MODEL, BLOCKED_PROMPT, guardrails=[name]) + result = poll_until_blocked( + lambda: client.chat(scoped_key, MODEL, BLOCKED_PROMPT, guardrails=[name]) + ) match result: case UnknownApiError(status_code=status, body=body): diff --git a/tests/e2e/guardrails/test_block_code_execution_guardrail_e2e.py b/tests/e2e/guardrails/test_block_code_execution_guardrail_e2e.py index de087b190d0..7cf4c195424 100644 --- a/tests/e2e/guardrails/test_block_code_execution_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_block_code_execution_guardrail_e2e.py @@ -14,9 +14,11 @@ the shared proxy, and the chat backend is a gemini deployment created for the te from __future__ import annotations +import time + import pytest -from e2e_config import unique_marker +from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, unique_marker from e2e_http import unwrap from guardrails_client import BlockCodeExecutionParamsBody, GuardrailsClient from lifecycle import ResourceManager @@ -54,7 +56,18 @@ class TestBlockCodeExecutionGuardrail: ) resources.defer(lambda: client.delete_guardrail(guardrail_id)) + # This guardrail replaces the reply rather than erroring, so wait for the + # block marker to appear instead of for a non-success status. The data-plane + # worker only picks a new guardrail up on its next DB sync (~30s), so the + # first call after the create is served without it. + deadline = time.monotonic() + POLL_TIMEOUT blocked = unwrap(client.chat(scoped_key, model, EXECUTION_REQUEST, guardrails=[name])) + while time.monotonic() < deadline: + if _BLOCK_MARKER in _first_content(blocked).lower(): + break + time.sleep(POLL_INTERVAL) + blocked = unwrap(client.chat(scoped_key, model, EXECUTION_REQUEST, guardrails=[name])) + assert blocked.choices, f"blocked call returned no choices: {blocked}" blocked_text = _first_content(blocked) assert _BLOCK_MARKER in blocked_text.lower(), ( diff --git a/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py b/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py index 39950259fb5..d117832221d 100644 --- a/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py @@ -16,7 +16,11 @@ import pytest from e2e_config import unique_marker from e2e_http import UnknownApiError, unwrap -from guardrails_client import GuardrailsClient, OpenAIModerationParamsBody +from guardrails_client import ( + GuardrailsClient, + OpenAIModerationParamsBody, + poll_until_blocked, +) from lifecycle import ResourceManager pytestmark = pytest.mark.e2e @@ -45,7 +49,9 @@ class TestOpenAIModerationGuardrail: ) resources.defer(lambda: client.delete_guardrail(guardrail_id)) - blocked = client.chat(scoped_key, model, FLAGGED_PROMPT, guardrails=[name]) + blocked = poll_until_blocked( + lambda: client.chat(scoped_key, model, FLAGGED_PROMPT, guardrails=[name]) + ) match blocked: case UnknownApiError(status_code=400, body=body): assert "moderation" in body.lower(), (