diff --git a/tests/e2e/coverage_registry/management_cases.py b/tests/e2e/coverage_registry/management_cases.py index 15dc7d333c5..812dbfe5b8d 100644 --- a/tests/e2e/coverage_registry/management_cases.py +++ b/tests/e2e/coverage_registry/management_cases.py @@ -63,7 +63,7 @@ MANAGEMENT_CASES: Final = tuple( node=f"{JWT_CLASS}::test_admin_creates_reads_updates_clears_and_deletes_a_key[virtual_key]", credential_kind="virtual_key", actor="proxy_admin", - profile="database_role", + profile="group_scoped", method="POST", path="/key/generate", operation_family="key_lifecycle", diff --git a/tests/e2e/idp.py b/tests/e2e/idp.py index 12db91bbd88..2dc7c2ad71b 100644 --- a/tests/e2e/idp.py +++ b/tests/e2e/idp.py @@ -8,6 +8,7 @@ import secrets import signal import subprocess import sys +import time import warnings from collections.abc import Callable from contextlib import ExitStack @@ -426,6 +427,36 @@ def token_claims(token: str) -> TokenClaims: return TokenClaims.model_validate_json(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4))) +def _signal_process_group(process_id: int, signum: int) -> bool: + try: + os.killpg(process_id, signum) + except ProcessLookupError: + return False + return True + + +def _stop_process_group(child: subprocess.Popen[bytes]) -> None: + _signal_process_group(child.pid, signal.SIGTERM) + deadline: Final = time.monotonic() + 5 + while _process_group_exists(child.pid): + child.poll() + if time.monotonic() >= deadline: + _signal_process_group(child.pid, signal.SIGKILL) + break + time.sleep(0.05) + child.wait() + + +def _process_group_exists(process_id: int) -> bool: + try: + os.killpg(process_id, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + def run_oidc_profile(proxy_url: str, command: list[str]) -> int: idp: Final = keycloak_from_env().with_strict_cleanup() with ExitStack() as cleanup: @@ -441,17 +472,11 @@ def run_oidc_profile(proxy_url: str, command: list[str]) -> int: client: Final = idp.browser_client(callback_url=f"{proxy_url.rstrip('/')}/sso/callback", defer=defer) environment: Final = {**os.environ, **client.environment(idp.discovery()), "PROXY_BASE_URL": proxy_url} - with subprocess.Popen(command, env=environment) as child: + with subprocess.Popen(command, env=environment, start_new_session=True) as child: try: return child.wait() finally: - if child.poll() is None: - child.terminate() - try: - child.wait(timeout=5) - except subprocess.TimeoutExpired: - child.kill() - child.wait() + _stop_process_group(child) if __name__ == "__main__": diff --git a/tests/e2e/management/jwt_actors.py b/tests/e2e/management/jwt_actors.py index 909d1652ada..2d23549fe71 100644 --- a/tests/e2e/management/jwt_actors.py +++ b/tests/e2e/management/jwt_actors.py @@ -84,7 +84,7 @@ class ActorFactory: ) ) ) - self.resources.defer(lambda: self.bootstrap.delete_key_strict(created.key)) + self.resources.defer(lambda: self.bootstrap.delete_key_strict(created.key, missing_ok=True)) return created def tenant(self) -> Tenant: diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py index a0243e868e2..8470d318db8 100644 --- a/tests/e2e/management/management_client.py +++ b/tests/e2e/management/management_client.py @@ -167,17 +167,18 @@ class ManagementClient: response_type=KeyInfoResponse, ) - def delete_key_strict(self, key: str, *, caller_key: str | None = None) -> None: + def delete_key_strict(self, key: str, *, caller_key: str | None = None, missing_ok: bool = False) -> None: """Strict delete for the act phase of a test: a failed delete is a hard failure, unlike the warn-only ProxyClient.delete_key used at teardown.""" - _ = unwrap( - self.proxy.transport.post( - "/key/delete", - headers=self.proxy.management_headers(caller_key), - json=KeyDeleteBody(keys=[key]), - response_type=NoBody, - ) + result = self.proxy.transport.post( + "/key/delete", + headers=self.proxy.management_headers(caller_key), + json=KeyDeleteBody(keys=[key]), + response_type=NoBody, ) + if missing_ok and isinstance(result, UnknownApiError) and result.status_code == 404: + return + _ = unwrap(result) def delete_model_strict(self, model_id: str) -> None: """Strict delete for the act phase of a test: a failed delete is a hard diff --git a/tests/e2e/management/test_jwt_management_e2e.py b/tests/e2e/management/test_jwt_management_e2e.py index 0d23f954158..5898073a4e6 100644 --- a/tests/e2e/management/test_jwt_management_e2e.py +++ b/tests/e2e/management/test_jwt_management_e2e.py @@ -102,31 +102,29 @@ class TestJwtManagement: @pytest.mark.parametrize("credential_kind", ("direct_jwt", "virtual_key")) def test_admin_creates_reads_updates_clears_and_deletes_a_key( self, - client: ManagementClient, - idp: Keycloak, - jwt_identity: Identity, - resources: ResourceManager, actor_factory: ActorFactory, credential_kind: Literal["direct_jwt", "virtual_key"], ) -> None: - actor: Final = actor_factory.create("proxy_admin") + tenant: Final = actor_factory.tenant() + actor: Final = actor_factory.create("proxy_admin", tenants=(tenant,), profile="group_scoped") virtual_key: Final = ( actor_factory.key(user_id=actor.identity.user_id).key if credential_kind == "virtual_key" else None ) - admin: Final = ( - virtual_key if virtual_key is not None else idp.access_token(jwt_identity, client_id=ADMIN_CLIENT_ID) + admin: Final = virtual_key if virtual_key is not None else actor.mint_caller(actor_factory.idp).credential + bound: Final = actor_factory.bootstrap.with_caller( + Caller(credential=admin, kind=credential_kind, role="proxy_admin") ) - bound: Final = client.with_caller(Caller(credential=admin, kind=credential_kind, role="proxy_admin")) + assert bound.user_info().user_id == actor.identity.user_id alias: Final = f"e2e-jwt-key-{unique_marker()}" created: Final = unwrap( bound.generate_key( - KeyGenerateBody(key_alias=alias, team_id=jwt_identity.group, models=[CHEAP_OPENAI_MODEL]), + KeyGenerateBody(key_alias=alias, team_id=tenant.team_id, models=[CHEAP_OPENAI_MODEL]), ) ) - resources.defer(lambda: client.proxy.delete_key(created.key)) + actor_factory.resources.defer(lambda: actor_factory.bootstrap.delete_key_strict(created.key, missing_ok=True)) original: Final = unwrap(bound.key_info_as(created.key)).info - assert original.key_alias == alias and original.team_id == jwt_identity.group + assert original.key_alias == alias and original.team_id == tenant.team_id assert original.models == [CHEAP_OPENAI_MODEL] updated_alias: Final = f"{alias}-updated" diff --git a/tests/e2e/test_idp.py b/tests/e2e/test_idp.py index 8a92dfa527c..cf2d4f3118a 100644 --- a/tests/e2e/test_idp.py +++ b/tests/e2e/test_idp.py @@ -6,6 +6,7 @@ from __future__ import annotations import os import signal +import socket import subprocess import sys import time @@ -148,16 +149,27 @@ def test_partial_provisioning_removes_the_group_when_user_creation_fails() -> No assert deletions.empty() -@pytest.mark.parametrize("exit_mode", ("normal", "parent", "group")) +@pytest.mark.parametrize( + ("exit_mode", "ignore_termination"), (("normal", False), ("parent", False), ("group", False), ("parent", True)) +) def test_oidc_launcher_removes_client_on_exit_and_termination( - tmp_path: Path, exit_mode: Literal["normal", "parent", "group"] + tmp_path: Path, exit_mode: Literal["normal", "parent", "group"], ignore_termination: bool ) -> None: ready: Final = tmp_path / "ready" + descendant_command: Final = ( + "import signal,socket,time; from pathlib import Path; " + + ("signal.signal(signal.SIGTERM, signal.SIG_IGN); " if ignore_termination else "") + + "listener=socket.socket(); listener.bind(('127.0.0.1',0)); listener.listen(); " + f"Path({str(ready)!r}).write_text(str(listener.getsockname()[1])); time.sleep(120)" + ) child_command: Final = ( - "import os,time; from pathlib import Path; " + "import os,subprocess,sys,time; from pathlib import Path; " 'assert os.environ["GENERIC_CLIENT_SECRET"]; ' 'assert os.environ["GENERIC_CLIENT_USE_PKCE"] == "true"; ' - f"Path({str(ready)!r}).touch(); " + ("raise SystemExit(7)" if exit_mode == "normal" else "time.sleep(120)") + f"subprocess.Popen([sys.executable, '-c', {descendant_command!r}]); " + f"ready=Path({str(ready)!r})\n" + "while not ready.exists(): time.sleep(0.05)\n" + + ("raise SystemExit(7)" if exit_mode == "normal" else "time.sleep(120)") ) with _idp_server() as (idp, deletions): with subprocess.Popen( @@ -187,7 +199,10 @@ def test_oidc_launcher_removes_client_on_exit_and_termination( process.terminate() elif exit_mode == "group": os.killpg(process.pid, signal.SIGTERM) - assert process.wait(timeout=10) == (7 if exit_mode == "normal" else 143) + assert process.wait(timeout=15) == (7 if exit_mode == "normal" else 143) + with socket.socket() as connection: + connection.settimeout(1) + assert connection.connect_ex(("127.0.0.1", int(ready.read_text()))) != 0 finally: if process.poll() is None: os.killpg(process.pid, signal.SIGKILL) diff --git a/tests/e2e/test_proxy_client.py b/tests/e2e/test_proxy_client.py index 879bd88980c..0c4aed5bd65 100644 --- a/tests/e2e/test_proxy_client.py +++ b/tests/e2e/test_proxy_client.py @@ -121,6 +121,13 @@ def caller_boundary( class TestBoundManagementCaller: + def test_strict_key_cleanup_accepts_missing_only_when_requested(self) -> None: + with caller_boundary(delete_status=404) as (bootstrap, received), without_retries(): + with pytest.raises(AssertionError): + bootstrap.delete_key_strict("owned") + bootstrap.delete_key_strict("owned", missing_ok=True) + assert (received.get_nowait(), received.get_nowait()) == ("Bearer bootstrap", "Bearer bootstrap") + def test_actor_key_cleanup_reports_failure_and_continues(self) -> None: with caller_boundary(delete_status=500) as (bootstrap, received), without_retries(): resources: Final = ResourceManager(client=bootstrap.proxy, strict_cleanup=True)