scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-04 02:31:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 70

fix(e2e): tolerate cross-pod guardrail sync delay in team-opt-out test

Browse source 
Stage runs multiple gateway pods behind the shared key. POST /guardrails
registers a new default-on guardrail in-process immediately only on the
pod that served the create call; every other pod picks it up on its next
periodic DB sync (proxy_server.py, every 30s), so the very next chat call
can race a pod that has not synced yet. Poll to a 40s deadline instead of
asserting on the first response, matching the existing pattern in
test_budget_reset_advances_e2e.py.
This commit is contained in:
mubashir1osmani 2026-07-21 11:31:59 -07:00
parent f67ac194a5
commit 3b58a9eba0
1 changed files with 34 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py
View file

@ -8,6 +8,8 @@ suite was removed.
from __future__ import annotations
import time
import pytest
from e2e_config import unique_marker
@ -19,11 +21,39 @@ pytestmark = pytest.mark.e2e
MODEL = "gemini-2.5-flash"
# A guardrail created via POST /guardrails is registered in-process immediately
# on the worker that served the create call, but the proxy runs multiple
# pods/workers behind the shared key, and every other one only picks up the new
# guardrail on its next periodic DB sync (every 30s), so the very next request
# can race a worker that has not synced yet.
GUARDRAIL_PROPAGATION_DEADLINE_SECONDS = 40.0
GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS = 5.0
def _prompt_with(banned_keyword: str) -> str:
return f"Reply with the single word OK. {banned_keyword}"
def _assert_eventually_blocked(client: GuardrailsClient, key: str, banned: str) -> None:
deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS
while True:
result = client.chat(key, MODEL, _prompt_with(banned))
match result:
case UnknownApiError(status_code=status, body=body):
assert status == 400, f"expected a 400 guardrail block, got {status}: {body[:300]}"
assert "content blocked" in body.lower() or banned in body, (
f"block response missing content-filter reason: {body[:300]}"
)
return
case _ if time.monotonic() < deadline:
time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS)
case _:
pytest.fail(
f"default-on guardrail never blocked the banned keyword within "
f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; got {result}"
)
class TestTeamDisableGlobalGuardrail:
@pytest.mark.covers(
"guardrail.litellm_content_filter.pre_call.blocks",
@ -33,25 +63,10 @@ class TestTeamDisableGlobalGuardrail:
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
banned = unique_marker()
guardrail_id = client.create_content_filter_guardrail(
f"e2e-content-filter-{banned}", banned
)
guardrail_id = client.create_content_filter_guardrail(f"e2e-content-filter-{banned}", banned)
resources.defer(lambda: client.delete_guardrail(guardrail_id))
result = client.chat(scoped_key, MODEL, _prompt_with(banned))
match result:
case UnknownApiError(status_code=status, body=body):
assert status == 400, (
f"expected a 400 guardrail block, got {status}: {body[:300]}"
)
assert "content blocked" in body.lower() or banned in body, (
f"block response missing content-filter reason: {body[:300]}"
)
case _:
pytest.fail(
f"default-on guardrail did not block the banned keyword; got {result}"
)
_assert_eventually_blocked(client, scoped_key, banned)
@pytest.mark.covers(
"guardrail.litellm_content_filter.pre_call.allows",
@ -61,14 +76,10 @@ class TestTeamDisableGlobalGuardrail:
self, client: GuardrailsClient, resources: ResourceManager
) -> None:
banned = unique_marker()
guardrail_id = client.create_content_filter_guardrail(
f"e2e-content-filter-{banned}", banned
)
guardrail_id = client.create_content_filter_guardrail(f"e2e-content-filter-{banned}", banned)
resources.defer(lambda: client.delete_guardrail(guardrail_id))
team_id = client.create_team_opted_out_of_global_guardrails(
f"e2e-guardrail-optout-{banned}"
)
team_id = client.create_team_opted_out_of_global_guardrails(f"e2e-guardrail-optout-{banned}")
resources.defer(lambda: client.delete_team(team_id))
key = client.create_key_in_team(team_id)
resources.defer(lambda: client.proxy.delete_key(key))