mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-11 08:38:24 +00:00
38b4d31447
* test(scim): audit #47544 Add integration cells for the SCIM group delete that now runs through the team delete's bulk path: an admin group delete revokes its members' admin key, a delete sent with the IdP's SCIM token revokes member keys and audits them under that token, a delete that times out on a locked member row changes nothing so a retry works, and two racing team deletes write one tombstone and one team audit row * test(scim): read the racing team delete's audit rows after the proxy drains them --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
531 lines
24 KiB
Python
531 lines
24 KiB
Python
"""DELETE /scim/v2/Groups/{id} deletes the group's team the way /team/delete does.
|
|
|
|
The members' group entries, their membership rows and the keys the members hold on the team go in one locked
|
|
pass whatever the roster size, and the configured admin group's members are demoted in that same pass, so they stay
|
|
admins until the group is gone. A member a concurrent group write added while the delete waited on the team lock is
|
|
demoted with the rest, and a group write that arrives once the delete holds the lock finds the group gone. The group
|
|
is 404 after, on every worker and on the peer proxy, and an unrelated key keeps serving.
|
|
A service-account key and a model on the team stay, so re-creating the group with the same id brings them back.
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import uuid
|
|
from collections.abc import Iterator
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from contextlib import contextmanager
|
|
from functools import partial
|
|
from hashlib import sha256
|
|
from pathlib import Path
|
|
from typing import Final
|
|
|
|
import httpx
|
|
import pytest
|
|
import yaml
|
|
from integration._support.client import GATEWAY_LIMITS, JSON_OBJECT, Gateway, eventually, object_value, string_value
|
|
from integration._support.database import read_rows
|
|
from integration._support.process import graceful_stop_seconds, owned_proxy_process
|
|
from integration._support.scim import (
|
|
GROUP_SCHEMA,
|
|
add_to_group,
|
|
assert_gone,
|
|
assert_landed,
|
|
body,
|
|
create_group,
|
|
created_team,
|
|
delete_group,
|
|
groups_of,
|
|
held_team_lock,
|
|
held_user_row,
|
|
keys_of,
|
|
member_ids,
|
|
membership_user_ids,
|
|
replace_group,
|
|
seed_users,
|
|
service_account_key,
|
|
team_key,
|
|
user_role,
|
|
waiters_on_lock_held_by,
|
|
waiters_on_rows_locked_by,
|
|
)
|
|
from pydantic import JsonValue, TypeAdapter
|
|
|
|
from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS, TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS
|
|
|
|
pytestmark: Final = pytest.mark.shares_worker_state
|
|
|
|
PROXY_ADMIN: Final = "proxy_admin"
|
|
DEMOTED_ROLE: Final = "internal_user_viewer"
|
|
MASTER_KEY_USER: Final = "default_user_id"
|
|
BURST_GROUPS: Final = 20
|
|
OWNED_PROXY_TIMEOUT: Final = int(2 * graceful_stop_seconds() + 120)
|
|
DELETED_TEAM_RECORDS_SQL: Final = 'SELECT team_id FROM "LiteLLM_DeletedTeamTable" WHERE team_id = %s'
|
|
DELETED_KEY_RECORDS_SQL: Final = 'SELECT token FROM "LiteLLM_DeletedVerificationToken" WHERE token = %s'
|
|
TEAM_MODEL_IDS_SQL: Final = """SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_info->>'team_id' = %s"""
|
|
BACKENDS_BLOCKED_BY_SQL: Final = "SELECT pid FROM pg_stat_activity WHERE %s::int = ANY(pg_blocking_pids(pid))"
|
|
USER_ROLE_SQL: Final = 'SELECT user_role FROM "LiteLLM_UserTable" WHERE user_id = %s'
|
|
DELETED_AUDIT_ROWS_SQL: Final = """
|
|
SELECT table_name, action, object_id, changed_by, changed_by_api_key
|
|
FROM "LiteLLM_AuditLog"
|
|
WHERE object_id IN (%s, %s) AND action = 'deleted'
|
|
ORDER BY table_name
|
|
"""
|
|
AUDIT_ROWS_NAMING_SQL: Final = """
|
|
SELECT table_name, object_id, changed_by_api_key
|
|
FROM "LiteLLM_AuditLog"
|
|
WHERE object_id = ANY(%s::text[]) AND action = 'deleted'
|
|
ORDER BY table_name, object_id
|
|
"""
|
|
OLDEST_TRANSACTION_BLOCKED_BY_SQL: Final = """
|
|
SELECT coalesce(max(extract(epoch FROM clock_timestamp() - xact_start)), 0)::float AS age
|
|
FROM pg_stat_activity
|
|
WHERE %s::int = ANY(pg_blocking_pids(pid))
|
|
"""
|
|
_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]")
|
|
_PIDS: Final = TypeAdapter(tuple[int, ...])
|
|
|
|
|
|
def _hashed(key: str) -> str:
|
|
return sha256(key.encode()).hexdigest()
|
|
|
|
|
|
def _admin_group_config(tmp_path: Path, admin_alias: str) -> Path:
|
|
config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
|
|
settings: Final = {**object_value(config["litellm_settings"]), "scim_admin_group": admin_alias}
|
|
path: Final = tmp_path / "scim-admin-group.yaml"
|
|
path.write_text(yaml.safe_dump({**config, "litellm_settings": settings}))
|
|
return path
|
|
|
|
|
|
def _roles_of(candidate: Gateway, users: tuple[str, ...]) -> frozenset[str]:
|
|
return frozenset(user_role(candidate, user) for user in users)
|
|
|
|
|
|
def _backends_blocked_by(holder: int) -> tuple[int, ...]:
|
|
return _PIDS.validate_python([row["pid"] for row in read_rows(BACKENDS_BLOCKED_BY_SQL, (str(holder),))])
|
|
|
|
|
|
def _stored_roles(users: tuple[str, ...]) -> frozenset[str]:
|
|
return frozenset(string_value(read_rows(USER_ROLE_SQL, (user,))[0]["user_role"]) for user in users)
|
|
|
|
|
|
def _models_status(candidate: Gateway, key: str) -> int:
|
|
return candidate.request("GET", "/v1/models", key=key).status_code
|
|
|
|
|
|
def _team_model_ids(team: str) -> frozenset[str]:
|
|
return frozenset(string_value(row["model_id"]) for row in read_rows(TEAM_MODEL_IDS_SQL, (team,)))
|
|
|
|
|
|
def _assert_revoked_everywhere(proxies: tuple[Gateway, ...], key: str) -> None:
|
|
for candidate in proxies:
|
|
eventually(partial(_models_status, candidate, key), lambda code: code == 401, seconds=30)
|
|
|
|
|
|
def _deleted_audit_rows(team: str, hashed_key: str) -> list[dict[str, JsonValue]]:
|
|
return read_rows(DELETED_AUDIT_ROWS_SQL, (team, hashed_key))
|
|
|
|
|
|
def _audit_rows_naming(object_ids: tuple[str, ...]) -> list[dict[str, JsonValue]]:
|
|
return read_rows(AUDIT_ROWS_NAMING_SQL, ("{" + ",".join(object_ids) + "}",))
|
|
|
|
|
|
def _oldest_transaction_blocked_by(holder: int) -> float:
|
|
age: Final = read_rows(OLDEST_TRANSACTION_BLOCKED_BY_SQL, (str(holder),))[0]["age"]
|
|
assert isinstance(age, float), age
|
|
return age
|
|
|
|
|
|
def _worker_startups(log: Path) -> tuple[tuple[int, ...], int]:
|
|
text: Final = log.read_text()
|
|
return _PIDS.validate_python(_STARTED_WORKER.findall(text)), text.count("Application startup complete.")
|
|
|
|
|
|
@contextmanager
|
|
def _fresh_connection(candidate: Gateway, timeout: httpx.Timeout | float | None = None) -> Iterator[Gateway]:
|
|
with httpx.Client(
|
|
base_url=candidate.client.base_url,
|
|
timeout=candidate.client.timeout if timeout is None else timeout,
|
|
trust_env=False,
|
|
limits=GATEWAY_LIMITS,
|
|
) as client:
|
|
yield Gateway(client, candidate.key, candidate.upstream_url)
|
|
|
|
|
|
def test_deleting_one_group_leaves_the_members_other_group_intact(gateway: Gateway) -> None:
|
|
with gateway.scenario() as scenario:
|
|
users: Final = seed_users(scenario, 3)
|
|
doomed: Final = created_team(scenario, create_group(gateway, users))
|
|
kept: Final = created_team(scenario, create_group(gateway, users))
|
|
kept_key: Final = team_key(gateway, kept, users[0])
|
|
|
|
deleted: Final = delete_group(gateway, doomed)
|
|
assert deleted.status_code == 204, deleted.text
|
|
|
|
assert gateway.request("GET", f"/scim/v2/Groups/{doomed}").status_code == 404
|
|
assert_landed(gateway, kept, users)
|
|
assert groups_of(gateway, users[0]) == frozenset({kept})
|
|
assert keys_of(kept) == frozenset({_hashed(kept_key)})
|
|
assert _models_status(gateway, kept_key) == 200
|
|
|
|
|
|
@pytest.mark.timeout(OWNED_PROXY_TIMEOUT)
|
|
def test_deleting_the_admin_group_demotes_its_members_and_writes_an_audit_row_naming_the_scim_caller(
|
|
gateway: Gateway, tmp_path: Path
|
|
) -> None:
|
|
admin_alias: Final = f"integration-admins-{uuid.uuid4().hex}"
|
|
with (
|
|
owned_proxy_process(
|
|
gateway,
|
|
tmp_path,
|
|
{"DATABASE_URL": os.environ["DATABASE_URL"], "LITELLM_STORE_AUDIT_LOGS": "true"},
|
|
config=_admin_group_config(tmp_path, admin_alias),
|
|
remove_environment=("DATABASE_URL_READ_REPLICA",),
|
|
workers=2,
|
|
) as owned,
|
|
owned.gateway.scenario() as scenario,
|
|
):
|
|
candidate: Final = owned.gateway
|
|
users: Final = seed_users(scenario, 3)
|
|
team: Final = created_team(scenario, create_group(candidate, users, display_name=admin_alias))
|
|
assert _roles_of(candidate, users) == frozenset({PROXY_ADMIN})
|
|
key: Final = team_key(candidate, team, users[0])
|
|
|
|
deleted: Final = delete_group(candidate, team)
|
|
assert deleted.status_code == 204, deleted.text
|
|
|
|
assert_gone(candidate, team, users, key)
|
|
assert _roles_of(candidate, users) == frozenset({DEMOTED_ROLE})
|
|
rows: Final = eventually(lambda: _deleted_audit_rows(team, _hashed(key)), lambda found: len(found) >= 2, 30)
|
|
caller: Final = {"changed_by": MASTER_KEY_USER, "changed_by_api_key": LITELLM_PROXY_MASTER_KEY_ALIAS}
|
|
assert rows == [
|
|
{"table_name": "LiteLLM_TeamTable", "action": "deleted", "object_id": team, **caller},
|
|
{"table_name": "LiteLLM_VerificationToken", "action": "deleted", "object_id": _hashed(key), **caller},
|
|
]
|
|
|
|
|
|
@pytest.mark.timeout(OWNED_PROXY_TIMEOUT)
|
|
def test_a_member_added_while_the_admin_group_delete_waits_on_the_team_lock_is_demoted_too(
|
|
gateway: Gateway, tmp_path: Path
|
|
) -> None:
|
|
admin_alias: Final = f"integration-admins-{uuid.uuid4().hex}"
|
|
with (
|
|
owned_proxy_process(
|
|
gateway,
|
|
tmp_path,
|
|
{"DATABASE_URL": os.environ["DATABASE_URL"]},
|
|
config=_admin_group_config(tmp_path, admin_alias),
|
|
remove_environment=("DATABASE_URL_READ_REPLICA",),
|
|
workers=2,
|
|
client_timeout=60,
|
|
) as owned,
|
|
owned.gateway.scenario() as scenario,
|
|
ThreadPoolExecutor(max_workers=2) as pool,
|
|
):
|
|
candidate: Final = owned.gateway
|
|
users: Final = seed_users(scenario, 5)
|
|
(late,) = seed_users(scenario, 1)
|
|
team: Final = created_team(scenario, create_group(candidate, users, display_name=admin_alias))
|
|
assert _roles_of(candidate, users) == frozenset({PROXY_ADMIN})
|
|
key: Final = team_key(candidate, team, users[0])
|
|
|
|
with held_team_lock(team) as holder:
|
|
put: Final = pool.submit(replace_group, candidate, team, (*users, late), display_name=admin_alias)
|
|
eventually(lambda: waiters_on_lock_held_by(holder), lambda waiting: waiting == 1, seconds=30)
|
|
delete: Final = pool.submit(delete_group, candidate, team)
|
|
eventually(lambda: waiters_on_lock_held_by(holder), lambda waiting: waiting == 2, seconds=30)
|
|
assert _stored_roles(users) == frozenset({PROXY_ADMIN})
|
|
replaced: Final = put.result()
|
|
deleted: Final = delete.result()
|
|
|
|
assert replaced.status_code == 200, replaced.text
|
|
assert member_ids(body(replaced)) == frozenset((*users, late))
|
|
assert deleted.status_code == 204, deleted.text
|
|
assert_gone(candidate, team, (*users, late), key)
|
|
assert _roles_of(candidate, (*users, late)) == frozenset({DEMOTED_ROLE})
|
|
|
|
|
|
@pytest.mark.timeout(OWNED_PROXY_TIMEOUT)
|
|
def test_a_member_added_while_the_admin_group_delete_holds_the_team_lock_is_refused_and_never_made_admin(
|
|
gateway: Gateway, tmp_path: Path
|
|
) -> None:
|
|
admin_alias: Final = f"integration-admins-{uuid.uuid4().hex}"
|
|
with (
|
|
owned_proxy_process(
|
|
gateway,
|
|
tmp_path,
|
|
{"DATABASE_URL": os.environ["DATABASE_URL"]},
|
|
config=_admin_group_config(tmp_path, admin_alias),
|
|
remove_environment=("DATABASE_URL_READ_REPLICA",),
|
|
workers=2,
|
|
client_timeout=60,
|
|
) as owned,
|
|
owned.gateway.scenario() as scenario,
|
|
ThreadPoolExecutor(max_workers=2) as pool,
|
|
):
|
|
candidate: Final = owned.gateway
|
|
users: Final = seed_users(scenario, 5)
|
|
(late,) = seed_users(scenario, 1)
|
|
team: Final = created_team(scenario, create_group(candidate, users, display_name=admin_alias))
|
|
assert _roles_of(candidate, users) == frozenset({PROXY_ADMIN})
|
|
key: Final = team_key(candidate, team, users[0])
|
|
|
|
with held_user_row(users[0]) as holder:
|
|
delete: Final = pool.submit(delete_group, candidate, team)
|
|
eventually(lambda: waiters_on_rows_locked_by(holder), lambda waiting: waiting == 1, seconds=30)
|
|
(deleting,) = _backends_blocked_by(holder)
|
|
added: Final = pool.submit(add_to_group, candidate, team, (late,))
|
|
eventually(lambda: _backends_blocked_by(deleting), lambda blocked: len(blocked) == 1, seconds=30)
|
|
assert _stored_roles(users) == frozenset({PROXY_ADMIN})
|
|
deleted: Final = delete.result()
|
|
|
|
assert deleted.status_code == 204, deleted.text
|
|
assert added.result().status_code == 404, added.result().text
|
|
assert_gone(candidate, team, users, key)
|
|
assert _roles_of(candidate, users) == frozenset({DEMOTED_ROLE})
|
|
assert user_role(candidate, late) != PROXY_ADMIN
|
|
|
|
|
|
def test_a_group_delete_revokes_member_keys_and_a_re_created_group_gets_its_service_account_key_and_model_back(
|
|
gateway: Gateway,
|
|
) -> None:
|
|
with gateway.scenario() as scenario:
|
|
users: Final = seed_users(scenario, 2)
|
|
team: Final = created_team(scenario, create_group(gateway, users))
|
|
member_key: Final = team_key(gateway, team, users[0])
|
|
service_key: Final = service_account_key(gateway, scenario, team)
|
|
scenario.model(model_info={"team_id": team})
|
|
team_models: Final = _team_model_ids(team)
|
|
assert team_models != frozenset()
|
|
|
|
deleted: Final = delete_group(gateway, team)
|
|
assert deleted.status_code == 204, deleted.text
|
|
|
|
assert gateway.request("GET", f"/scim/v2/Groups/{team}").status_code == 404
|
|
assert membership_user_ids(team) == frozenset()
|
|
assert keys_of(team) == frozenset({_hashed(service_key)})
|
|
assert _team_model_ids(team) == team_models
|
|
_assert_revoked_everywhere((gateway,), member_key)
|
|
|
|
recreated: Final = gateway.request(
|
|
"POST",
|
|
"/scim/v2/Groups",
|
|
{"schemas": [GROUP_SCHEMA], "externalId": team, "displayName": f"integration-{uuid.uuid4().hex}"},
|
|
)
|
|
assert recreated.status_code == 201, recreated.text
|
|
assert string_value(body(recreated)["id"]) == team
|
|
eventually(partial(_models_status, gateway, service_key), lambda code: code == 200, seconds=90)
|
|
assert _models_status(gateway, member_key) == 401
|
|
|
|
|
|
@pytest.mark.timeout(OWNED_PROXY_TIMEOUT)
|
|
def test_a_group_delete_sent_with_the_idps_scim_token_revokes_member_keys_and_audits_them_under_that_token(
|
|
gateway: Gateway, tmp_path: Path
|
|
) -> None:
|
|
with (
|
|
owned_proxy_process(
|
|
gateway,
|
|
tmp_path,
|
|
{"DATABASE_URL": os.environ["DATABASE_URL"], "LITELLM_STORE_AUDIT_LOGS": "true"},
|
|
remove_environment=("DATABASE_URL_READ_REPLICA",),
|
|
) as owned,
|
|
owned.gateway.scenario() as scenario,
|
|
):
|
|
candidate: Final = owned.gateway
|
|
scim_token: Final = scenario.key(allowed_routes=["/scim/*"], models=[])
|
|
users: Final = seed_users(scenario, 3)
|
|
team: Final = created_team(scenario, create_group(candidate, users))
|
|
member_keys: Final = (team_key(candidate, team, users[0]), team_key(candidate, team, users[1]))
|
|
service_key: Final = service_account_key(candidate, scenario, team)
|
|
|
|
deleted: Final = delete_group(candidate, team, key=scim_token)
|
|
assert deleted.status_code == 204, deleted.text
|
|
|
|
assert keys_of(team) == frozenset({_hashed(service_key)})
|
|
for key in member_keys:
|
|
_assert_revoked_everywhere((candidate,), key)
|
|
audited: Final = (team, *(_hashed(key) for key in member_keys), _hashed(service_key))
|
|
rows: Final = eventually(lambda: _audit_rows_naming(audited), lambda found: len(found) >= 3, seconds=30)
|
|
token: Final = _hashed(scim_token)
|
|
assert rows == [
|
|
{"table_name": "LiteLLM_TeamTable", "object_id": team, "changed_by_api_key": token},
|
|
*(
|
|
{"table_name": "LiteLLM_VerificationToken", "object_id": hashed, "changed_by_api_key": token}
|
|
for hashed in sorted(_hashed(key) for key in member_keys)
|
|
),
|
|
]
|
|
|
|
|
|
@pytest.mark.timeout(TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS + 120)
|
|
def test_a_group_delete_that_outlives_its_transaction_on_a_locked_member_row_changes_nothing_and_a_retry_lands(
|
|
gateway: Gateway,
|
|
) -> None:
|
|
patient_timeout: Final = TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS + 90
|
|
with (
|
|
gateway.scenario() as scenario,
|
|
_fresh_connection(gateway, timeout=patient_timeout) as patient,
|
|
ThreadPoolExecutor(max_workers=1) as pool,
|
|
):
|
|
users: Final = seed_users(scenario, 3)
|
|
team: Final = created_team(scenario, create_group(gateway, users))
|
|
key: Final = team_key(gateway, team, users[0])
|
|
|
|
with held_user_row(users[0]) as holder:
|
|
delete: Final = pool.submit(delete_group, patient, team)
|
|
eventually(
|
|
lambda: (
|
|
delete.done()
|
|
or _oldest_transaction_blocked_by(holder) > TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS + 1
|
|
),
|
|
lambda outlived: outlived,
|
|
seconds=TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS + 30,
|
|
)
|
|
failed: Final = delete.result(timeout=patient_timeout)
|
|
|
|
assert failed.status_code >= 500, failed.text
|
|
assert_landed(gateway, team, users)
|
|
assert keys_of(team) == frozenset({_hashed(key)})
|
|
assert read_rows(DELETED_TEAM_RECORDS_SQL, (team,)) == []
|
|
assert read_rows(DELETED_KEY_RECORDS_SQL, (_hashed(key),)) == []
|
|
assert _models_status(gateway, key) == 200
|
|
|
|
retried: Final = delete_group(gateway, team)
|
|
assert retried.status_code == 204, retried.text
|
|
assert_gone(gateway, team, users, key)
|
|
|
|
|
|
def test_deleting_an_unknown_group_is_404(gateway: Gateway) -> None:
|
|
missing: Final = f"missing-{uuid.uuid4().hex}"
|
|
response: Final = delete_group(gateway, missing)
|
|
assert response.status_code == 404, response.text
|
|
assert "error" in body(response), response.text
|
|
assert missing in response.text
|
|
|
|
|
|
def test_deleting_a_group_twice_is_404_the_second_time(gateway: Gateway) -> None:
|
|
with gateway.scenario() as scenario:
|
|
users: Final = seed_users(scenario, 2)
|
|
team: Final = created_team(scenario, create_group(gateway, users))
|
|
first: Final = delete_group(gateway, team)
|
|
assert first.status_code == 204, first.text
|
|
second: Final = delete_group(gateway, team)
|
|
assert second.status_code == 404, second.text
|
|
assert "error" in body(second), second.text
|
|
assert gateway.request("GET", f"/scim/v2/Groups/{team}").status_code == 404
|
|
assert membership_user_ids(team) == frozenset()
|
|
|
|
|
|
def test_two_deletes_racing_on_one_group_answer_204_then_404_and_record_the_team_once(gateway: Gateway) -> None:
|
|
with gateway.scenario() as scenario, ThreadPoolExecutor(max_workers=2) as pool:
|
|
users: Final = seed_users(scenario, 2)
|
|
team: Final = created_team(scenario, create_group(gateway, users))
|
|
key: Final = team_key(gateway, team, users[0])
|
|
|
|
with held_team_lock(team) as holder:
|
|
deletes: Final = (pool.submit(delete_group, gateway, team), pool.submit(delete_group, gateway, team))
|
|
eventually(lambda: waiters_on_lock_held_by(holder), lambda waiting: waiting == 2, seconds=30)
|
|
|
|
assert sorted(delete.result().status_code for delete in deletes) == [204, 404]
|
|
assert len(read_rows(DELETED_TEAM_RECORDS_SQL, (team,))) == 1
|
|
assert len(read_rows(DELETED_KEY_RECORDS_SQL, (_hashed(key),))) == 1
|
|
assert gateway.request("GET", f"/scim/v2/Groups/{team}").status_code == 404
|
|
|
|
|
|
def test_deleting_a_group_with_a_non_admin_key_is_refused(gateway: Gateway) -> None:
|
|
with gateway.scenario() as scenario:
|
|
users: Final = seed_users(scenario, 2)
|
|
team: Final = created_team(scenario, create_group(gateway, users))
|
|
outsider: Final = scenario.key()
|
|
response: Final = delete_group(gateway, team, key=outsider)
|
|
assert response.status_code in (401, 403), response.text
|
|
assert "error" in body(response), response.text
|
|
assert_landed(gateway, team, users)
|
|
|
|
|
|
@pytest.mark.parametrize("group_id", ("123", "x" * 5000, "a%2Fb"), ids=("int_like", "five_kb", "encoded_slash"))
|
|
def test_deleting_a_group_with_a_hostile_id_is_404(gateway: Gateway, group_id: str) -> None:
|
|
response: Final = delete_group(gateway, group_id)
|
|
assert response.status_code == 404, response.text
|
|
assert isinstance(response.json(), dict), response.text
|
|
assert gateway.request("GET", "/health/liveliness").status_code == 200
|
|
|
|
|
|
def test_deleting_an_empty_group_is_204_then_404(gateway: Gateway) -> None:
|
|
with gateway.scenario() as scenario:
|
|
team: Final = created_team(scenario, create_group(gateway, ()))
|
|
assert member_ids(gateway.get(f"/scim/v2/Groups/{team}")) == frozenset()
|
|
deleted: Final = delete_group(gateway, team)
|
|
assert deleted.status_code == 204, deleted.text
|
|
assert gateway.request("GET", f"/scim/v2/Groups/{team}").status_code == 404
|
|
assert membership_user_ids(team) == frozenset()
|
|
|
|
|
|
def test_group_delete_on_one_proxy_revokes_the_team_key_and_the_group_on_the_peer(
|
|
gateway: Gateway, peer: Gateway
|
|
) -> None:
|
|
with gateway.scenario() as scenario:
|
|
users: Final = seed_users(scenario, 3)
|
|
team: Final = created_team(scenario, create_group(gateway, users))
|
|
key: Final = team_key(gateway, team, users[0])
|
|
assert _models_status(peer, key) == 200
|
|
assert member_ids(peer.get(f"/scim/v2/Groups/{team}")) == frozenset(users)
|
|
|
|
deleted: Final = delete_group(gateway, team)
|
|
assert deleted.status_code == 204, deleted.text
|
|
|
|
assert peer.request("GET", f"/scim/v2/Groups/{team}").status_code == 404
|
|
_assert_revoked_everywhere((peer, gateway), key)
|
|
assert_gone(peer, team, users, key)
|
|
|
|
|
|
def test_twenty_group_deletes_split_across_two_proxies_all_land(gateway: Gateway, peer: Gateway) -> None:
|
|
proxies: Final = (gateway, peer)
|
|
with gateway.scenario() as scenario, ThreadPoolExecutor(max_workers=10) as pool:
|
|
bystander: Final = scenario.key()
|
|
rosters: Final = tuple(seed_users(scenario, 3) for _ in range(BURST_GROUPS))
|
|
teams: Final = tuple(created_team(scenario, create_group(gateway, users)) for users in rosters)
|
|
keys: Final = tuple(team_key(gateway, team, users[0]) for team, users in zip(teams, rosters))
|
|
|
|
assigned: Final = tuple(proxies[index % 2] for index in range(BURST_GROUPS))
|
|
responses: Final = tuple(pool.map(delete_group, assigned, teams))
|
|
|
|
for response in responses:
|
|
assert response.status_code == 204, response.text
|
|
for team, users, key in zip(teams, rosters, keys):
|
|
_assert_revoked_everywhere(proxies, key)
|
|
assert_gone(gateway, team, users, key)
|
|
assert peer.request("GET", f"/scim/v2/Groups/{team}").status_code == 404
|
|
assert _models_status(gateway, bystander) == 200
|
|
assert _models_status(peer, bystander) == 200
|
|
|
|
|
|
@pytest.mark.timeout(OWNED_PROXY_TIMEOUT)
|
|
def test_group_delete_after_a_worker_was_killed_still_completes(gateway: Gateway, tmp_path: Path) -> None:
|
|
with (
|
|
owned_proxy_process(
|
|
gateway, tmp_path, {}, remove_environment=("DATABASE_URL_READ_REPLICA",), workers=2
|
|
) as owned,
|
|
owned.gateway.scenario() as scenario,
|
|
):
|
|
candidate: Final = owned.gateway
|
|
workers, _ = eventually(
|
|
lambda: _worker_startups(owned.log), lambda found: len(found[0]) == 2 and found[1] == 2, seconds=120
|
|
)
|
|
first_users: Final = seed_users(scenario, 3)
|
|
second_users: Final = seed_users(scenario, 3)
|
|
first: Final = created_team(scenario, create_group(candidate, first_users))
|
|
second: Final = created_team(scenario, create_group(candidate, second_users))
|
|
first_key: Final = team_key(candidate, first, first_users[0])
|
|
second_key: Final = team_key(candidate, second, second_users[0])
|
|
|
|
with _fresh_connection(candidate) as survivor:
|
|
with owned.replacing_worker(workers[0]) as kill:
|
|
kill.freeze()
|
|
kill()
|
|
deleted_first: Final = delete_group(survivor, first)
|
|
assert deleted_first.status_code == 204, deleted_first.text
|
|
assert_gone(survivor, first, first_users, first_key)
|
|
|
|
deleted_second: Final = delete_group(survivor, second)
|
|
assert deleted_second.status_code == 204, deleted_second.text
|
|
assert_gone(survivor, second, second_users, second_key)
|