From 171660ab962d2065176a743ab661dee458695255 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 18 Aug 2026 08:51:30 -0700 Subject: [PATCH] test(e2e): assert the guardrail enforces before asserting a team bypasses it test_team_with_disable_flag_bypasses_global_guardrail created a default-on content filter and asserted that a key on a team with disable_global_guardrails got a successful response. It never checked the guardrail was enforcing, so any reason it was inert (keyword never matching, default_on not applied, a partial create) made the call succeed for the wrong reason and the test pass. A completely broken disable_global_guardrails stayed green. Prove enforcement on a key outside the opted-out team first, then assert the bypass, and check the response is the model's own answer rather than the content-blocked message by requiring the model was actually invoked. That mirrors what the block_code_execution suite already does, and it holds up on a thinking model where a low max_tokens budget can leave the text empty. Verified against a live proxy on real gemini-2.5-flash: both tests pass, and with the guardrail registered under a keyword absent from the prompt the old shape still passed while the new one fails on the enforcement check. --- .../test_team_disable_global_guardrail_e2e.py | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py b/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py index db917d6ede9..80d250f8bac 100644 --- a/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py @@ -16,24 +16,30 @@ from e2e_config import unique_marker from e2e_http import UnknownApiError, unwrap from guardrails_client import GuardrailsClient from lifecycle import ResourceManager +from models import ChatResponse pytestmark = pytest.mark.e2e MODEL = "gemini-2.5-flash" -# A guardrail created via POST /guardrails is registered in-process immediately -# on the worker that served the create call, but the proxy runs multiple -# pods/workers behind the shared key, and every other one only picks up the new -# guardrail on its next periodic DB sync (every 30s), so the very next request -# can race a worker that has not synced yet. +# register() already settles the config reload, but a replica that missed its window serves the old config. GUARDRAIL_PROPAGATION_DEADLINE_SECONDS = 40.0 GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS = 5.0 +_BLOCK_MARKER = "content blocked" + def _prompt_with(banned_keyword: str) -> str: return f"Reply with the single word OK. {banned_keyword}" +def _first_content(response: ChatResponse) -> str: + if not response.choices: + return "" + message = response.choices[0].message + return (message.content if message else None) or "" + + def _assert_eventually_blocked(client: GuardrailsClient, key: str, banned: str) -> None: deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS while True: @@ -41,7 +47,7 @@ def _assert_eventually_blocked(client: GuardrailsClient, key: str, banned: str) match result: case UnknownApiError(status_code=status, body=body): assert status == 400, f"expected a 400 guardrail block, got {status}: {body[:300]}" - assert "content blocked" in body.lower() or banned in body, ( + assert _BLOCK_MARKER in body.lower() or banned in body, ( f"block response missing content-filter reason: {body[:300]}" ) return @@ -73,7 +79,7 @@ class TestTeamDisableGlobalGuardrail: exercised_on=["chat_completions"], ) def test_team_with_disable_flag_bypasses_global_guardrail( - self, client: GuardrailsClient, resources: ResourceManager + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: banned = unique_marker() guardrail_id = client.create_content_filter_guardrail(f"e2e-content-filter-{banned}", banned) @@ -84,9 +90,20 @@ class TestTeamDisableGlobalGuardrail: key = client.create_key_in_team(team_id) resources.defer(lambda: client.proxy.delete_key(key)) - chat = unwrap(client.chat(key, MODEL, _prompt_with(banned))) + _assert_eventually_blocked(client, scoped_key, banned) + + chat = unwrap(client.chat(key, MODEL, _prompt_with(banned), max_tokens=256)) assert chat.choices, ( f"team opted out of global guardrails, so the banned keyword must pass " f"through and the call must succeed, but no choices came back: {chat}" ) + text = _first_content(chat) + assert _BLOCK_MARKER not in text.lower(), ( + f"the guardrail intercepted a key on a team opted out of global guardrails, " + f"returning the content-blocked message instead of the model's answer: {text[:300]!r}" + ) + assert chat.usage is not None and (chat.usage.prompt_tokens or 0) > 0, ( + f"the opted-out call must reach the model, but the model was never " + f"invoked; usage was {chat.usage}" + )