test: enforce isolated actors and stop OIDC process groups

tests/e2e/coverage_registry/management_cases.py

@@ -63,7 +63,7 @@ MANAGEMENT_CASES: Final = tuple(
         node=f"{JWT_CLASS}::test_admin_creates_reads_updates_clears_and_deletes_a_key[virtual_key]",
         credential_kind="virtual_key",
         actor="proxy_admin",
-        profile="database_role",
+        profile="group_scoped",
         method="POST",
         path="/key/generate",
         operation_family="key_lifecycle",

tests/e2e/idp.py

@@ -8,6 +8,7 @@ import secrets
 import signal
 import subprocess
 import sys
+import time
 import warnings
 from collections.abc import Callable
 from contextlib import ExitStack
@@ -426,6 +427,36 @@ def token_claims(token: str) -> TokenClaims:
     return TokenClaims.model_validate_json(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4)))
 
 
+def _signal_process_group(process_id: int, signum: int) -> bool:
+    try:
+        os.killpg(process_id, signum)
+    except ProcessLookupError:
+        return False
+    return True
+
+
+def _stop_process_group(child: subprocess.Popen[bytes]) -> None:
+    _signal_process_group(child.pid, signal.SIGTERM)
+    deadline: Final = time.monotonic() + 5
+    while _process_group_exists(child.pid):
+        child.poll()
+        if time.monotonic() >= deadline:
+            _signal_process_group(child.pid, signal.SIGKILL)
+            break
+        time.sleep(0.05)
+    child.wait()
+
+
+def _process_group_exists(process_id: int) -> bool:
+    try:
+        os.killpg(process_id, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    return True
+
+
 def run_oidc_profile(proxy_url: str, command: list[str]) -> int:
     idp: Final = keycloak_from_env().with_strict_cleanup()
     with ExitStack() as cleanup:
@@ -441,17 +472,11 @@ def run_oidc_profile(proxy_url: str, command: list[str]) -> int:
 
         client: Final = idp.browser_client(callback_url=f"{proxy_url.rstrip('/')}/sso/callback", defer=defer)
         environment: Final = {**os.environ, **client.environment(idp.discovery()), "PROXY_BASE_URL": proxy_url}
-        with subprocess.Popen(command, env=environment) as child:
+        with subprocess.Popen(command, env=environment, start_new_session=True) as child:
             try:
                 return child.wait()
             finally:
-                if child.poll() is None:
-                    child.terminate()
-                    try:
-                        child.wait(timeout=5)
-                    except subprocess.TimeoutExpired:
-                        child.kill()
-                        child.wait()
+                _stop_process_group(child)
 
 
 if __name__ == "__main__":

tests/e2e/management/jwt_actors.py

@@ -84,7 +84,7 @@ class ActorFactory:
                 )
             )
         )
-        self.resources.defer(lambda: self.bootstrap.delete_key_strict(created.key))
+        self.resources.defer(lambda: self.bootstrap.delete_key_strict(created.key, missing_ok=True))
         return created
 
     def tenant(self) -> Tenant:

tests/e2e/management/management_client.py

@@ -167,17 +167,18 @@ class ManagementClient:
             response_type=KeyInfoResponse,
         )
 
-    def delete_key_strict(self, key: str, *, caller_key: str | None = None) -> None:
+    def delete_key_strict(self, key: str, *, caller_key: str | None = None, missing_ok: bool = False) -> None:
         """Strict delete for the act phase of a test: a failed delete is a hard
         failure, unlike the warn-only ProxyClient.delete_key used at teardown."""
-        _ = unwrap(
-            self.proxy.transport.post(
-                "/key/delete",
-                headers=self.proxy.management_headers(caller_key),
-                json=KeyDeleteBody(keys=[key]),
-                response_type=NoBody,
-            )
+        result = self.proxy.transport.post(
+            "/key/delete",
+            headers=self.proxy.management_headers(caller_key),
+            json=KeyDeleteBody(keys=[key]),
+            response_type=NoBody,
         )
+        if missing_ok and isinstance(result, UnknownApiError) and result.status_code == 404:
+            return
+        _ = unwrap(result)
 
     def delete_model_strict(self, model_id: str) -> None:
         """Strict delete for the act phase of a test: a failed delete is a hard

tests/e2e/management/test_jwt_management_e2e.py

@@ -102,31 +102,29 @@ class TestJwtManagement:
     @pytest.mark.parametrize("credential_kind", ("direct_jwt", "virtual_key"))
     def test_admin_creates_reads_updates_clears_and_deletes_a_key(
         self,
-        client: ManagementClient,
-        idp: Keycloak,
-        jwt_identity: Identity,
-        resources: ResourceManager,
         actor_factory: ActorFactory,
         credential_kind: Literal["direct_jwt", "virtual_key"],
     ) -> None:
-        actor: Final = actor_factory.create("proxy_admin")
+        tenant: Final = actor_factory.tenant()
+        actor: Final = actor_factory.create("proxy_admin", tenants=(tenant,), profile="group_scoped")
         virtual_key: Final = (
             actor_factory.key(user_id=actor.identity.user_id).key if credential_kind == "virtual_key" else None
         )
-        admin: Final = (
-            virtual_key if virtual_key is not None else idp.access_token(jwt_identity, client_id=ADMIN_CLIENT_ID)
+        admin: Final = virtual_key if virtual_key is not None else actor.mint_caller(actor_factory.idp).credential
+        bound: Final = actor_factory.bootstrap.with_caller(
+            Caller(credential=admin, kind=credential_kind, role="proxy_admin")
         )
-        bound: Final = client.with_caller(Caller(credential=admin, kind=credential_kind, role="proxy_admin"))
+        assert bound.user_info().user_id == actor.identity.user_id
         alias: Final = f"e2e-jwt-key-{unique_marker()}"
         created: Final = unwrap(
             bound.generate_key(
-                KeyGenerateBody(key_alias=alias, team_id=jwt_identity.group, models=[CHEAP_OPENAI_MODEL]),
+                KeyGenerateBody(key_alias=alias, team_id=tenant.team_id, models=[CHEAP_OPENAI_MODEL]),
             )
         )
-        resources.defer(lambda: client.proxy.delete_key(created.key))
+        actor_factory.resources.defer(lambda: actor_factory.bootstrap.delete_key_strict(created.key, missing_ok=True))
 
         original: Final = unwrap(bound.key_info_as(created.key)).info
-        assert original.key_alias == alias and original.team_id == jwt_identity.group
+        assert original.key_alias == alias and original.team_id == tenant.team_id
         assert original.models == [CHEAP_OPENAI_MODEL]
 
         updated_alias: Final = f"{alias}-updated"

tests/e2e/test_idp.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 
 import os
 import signal
+import socket
 import subprocess
 import sys
 import time
@@ -148,16 +149,27 @@ def test_partial_provisioning_removes_the_group_when_user_creation_fails() -> No
         assert deletions.empty()
 
 
-@pytest.mark.parametrize("exit_mode", ("normal", "parent", "group"))
+@pytest.mark.parametrize(
+    ("exit_mode", "ignore_termination"), (("normal", False), ("parent", False), ("group", False), ("parent", True))
+)
 def test_oidc_launcher_removes_client_on_exit_and_termination(
-    tmp_path: Path, exit_mode: Literal["normal", "parent", "group"]
+    tmp_path: Path, exit_mode: Literal["normal", "parent", "group"], ignore_termination: bool
 ) -> None:
     ready: Final = tmp_path / "ready"
+    descendant_command: Final = (
+        "import signal,socket,time; from pathlib import Path; "
+        + ("signal.signal(signal.SIGTERM, signal.SIG_IGN); " if ignore_termination else "")
+        + "listener=socket.socket(); listener.bind(('127.0.0.1',0)); listener.listen(); "
+        f"Path({str(ready)!r}).write_text(str(listener.getsockname()[1])); time.sleep(120)"
+    )
     child_command: Final = (
-        "import os,time; from pathlib import Path; "
+        "import os,subprocess,sys,time; from pathlib import Path; "
         'assert os.environ["GENERIC_CLIENT_SECRET"]; '
         'assert os.environ["GENERIC_CLIENT_USE_PKCE"] == "true"; '
-        f"Path({str(ready)!r}).touch(); " + ("raise SystemExit(7)" if exit_mode == "normal" else "time.sleep(120)")
+        f"subprocess.Popen([sys.executable, '-c', {descendant_command!r}]); "
+        f"ready=Path({str(ready)!r})\n"
+        "while not ready.exists(): time.sleep(0.05)\n"
+        + ("raise SystemExit(7)" if exit_mode == "normal" else "time.sleep(120)")
     )
     with _idp_server() as (idp, deletions):
         with subprocess.Popen(
@@ -187,7 +199,10 @@ def test_oidc_launcher_removes_client_on_exit_and_termination(
                     process.terminate()
                 elif exit_mode == "group":
                     os.killpg(process.pid, signal.SIGTERM)
-                assert process.wait(timeout=10) == (7 if exit_mode == "normal" else 143)
+                assert process.wait(timeout=15) == (7 if exit_mode == "normal" else 143)
+                with socket.socket() as connection:
+                    connection.settimeout(1)
+                    assert connection.connect_ex(("127.0.0.1", int(ready.read_text()))) != 0
             finally:
                 if process.poll() is None:
                     os.killpg(process.pid, signal.SIGKILL)

tests/e2e/test_proxy_client.py

@@ -121,6 +121,13 @@ def caller_boundary(
 
 
 class TestBoundManagementCaller:
+    def test_strict_key_cleanup_accepts_missing_only_when_requested(self) -> None:
+        with caller_boundary(delete_status=404) as (bootstrap, received), without_retries():
+            with pytest.raises(AssertionError):
+                bootstrap.delete_key_strict("owned")
+            bootstrap.delete_key_strict("owned", missing_ok=True)
+            assert (received.get_nowait(), received.get_nowait()) == ("Bearer bootstrap", "Bearer bootstrap")
+
     def test_actor_key_cleanup_reports_failure_and_continues(self) -> None:
         with caller_boundary(delete_status=500) as (bootstrap, received), without_retries():
             resources: Final = ResourceManager(client=bootstrap.proxy, strict_cleanup=True)