From 3b58a9eba0898583878f08cee0dd8d7637a2b208 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 21 Jul 2026 11:31:59 -0700 Subject: [PATCH] fix(e2e): tolerate cross-pod guardrail sync delay in team-opt-out test Stage runs multiple gateway pods behind the shared key. POST /guardrails registers a new default-on guardrail in-process immediately only on the pod that served the create call; every other pod picks it up on its next periodic DB sync (proxy_server.py, every 30s), so the very next chat call can race a pod that has not synced yet. Poll to a 40s deadline instead of asserting on the first response, matching the existing pattern in test_budget_reset_advances_e2e.py. --- .../test_team_disable_global_guardrail_e2e.py | 57 +++++++++++-------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py b/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py index cd32a19d54a..db917d6ede9 100644 --- a/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py @@ -8,6 +8,8 @@ suite was removed. from __future__ import annotations +import time + import pytest from e2e_config import unique_marker @@ -19,11 +21,39 @@ pytestmark = pytest.mark.e2e MODEL = "gemini-2.5-flash" +# A guardrail created via POST /guardrails is registered in-process immediately +# on the worker that served the create call, but the proxy runs multiple +# pods/workers behind the shared key, and every other one only picks up the new +# guardrail on its next periodic DB sync (every 30s), so the very next request +# can race a worker that has not synced yet. +GUARDRAIL_PROPAGATION_DEADLINE_SECONDS = 40.0 +GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS = 5.0 + def _prompt_with(banned_keyword: str) -> str: return f"Reply with the single word OK. {banned_keyword}" +def _assert_eventually_blocked(client: GuardrailsClient, key: str, banned: str) -> None: + deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS + while True: + result = client.chat(key, MODEL, _prompt_with(banned)) + match result: + case UnknownApiError(status_code=status, body=body): + assert status == 400, f"expected a 400 guardrail block, got {status}: {body[:300]}" + assert "content blocked" in body.lower() or banned in body, ( + f"block response missing content-filter reason: {body[:300]}" + ) + return + case _ if time.monotonic() < deadline: + time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS) + case _: + pytest.fail( + f"default-on guardrail never blocked the banned keyword within " + f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; got {result}" + ) + + class TestTeamDisableGlobalGuardrail: @pytest.mark.covers( "guardrail.litellm_content_filter.pre_call.blocks", @@ -33,25 +63,10 @@ class TestTeamDisableGlobalGuardrail: self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: banned = unique_marker() - guardrail_id = client.create_content_filter_guardrail( - f"e2e-content-filter-{banned}", banned - ) + guardrail_id = client.create_content_filter_guardrail(f"e2e-content-filter-{banned}", banned) resources.defer(lambda: client.delete_guardrail(guardrail_id)) - result = client.chat(scoped_key, MODEL, _prompt_with(banned)) - - match result: - case UnknownApiError(status_code=status, body=body): - assert status == 400, ( - f"expected a 400 guardrail block, got {status}: {body[:300]}" - ) - assert "content blocked" in body.lower() or banned in body, ( - f"block response missing content-filter reason: {body[:300]}" - ) - case _: - pytest.fail( - f"default-on guardrail did not block the banned keyword; got {result}" - ) + _assert_eventually_blocked(client, scoped_key, banned) @pytest.mark.covers( "guardrail.litellm_content_filter.pre_call.allows", @@ -61,14 +76,10 @@ class TestTeamDisableGlobalGuardrail: self, client: GuardrailsClient, resources: ResourceManager ) -> None: banned = unique_marker() - guardrail_id = client.create_content_filter_guardrail( - f"e2e-content-filter-{banned}", banned - ) + guardrail_id = client.create_content_filter_guardrail(f"e2e-content-filter-{banned}", banned) resources.defer(lambda: client.delete_guardrail(guardrail_id)) - team_id = client.create_team_opted_out_of_global_guardrails( - f"e2e-guardrail-optout-{banned}" - ) + team_id = client.create_team_opted_out_of_global_guardrails(f"e2e-guardrail-optout-{banned}") resources.defer(lambda: client.delete_team(team_id)) key = client.create_key_in_team(team_id) resources.defer(lambda: client.proxy.delete_key(key))