From ede3637bf6774c9807a44a388bb981b1cbd68ac1 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 21 Jul 2026 10:32:55 -0700 Subject: [PATCH] test(e2e): add other suite covering master-key auth and health lifecycle Covers the other.* holding-pen cells that were uncovered: master-key valid_allows/invalid_denied on the admin /user/list gate, and the lifecycle probes liveness.ping, readiness.public_probe, readiness.reports_db_status, and readiness_details.authenticated_diagnostics. New tests/e2e/other/ suite on the shared ProxyClient; the health probes send no auth header to prove the public routes need no credential, and the details route is asserted to reject an anonymous caller while exposing version/db diagnostics to the master key. --- tests/e2e/CLAUDE.md | 1 + tests/e2e/models.py | 20 ++++++ tests/e2e/other/conftest.py | 18 +++++ tests/e2e/other/other_client.py | 73 ++++++++++++++++++++ tests/e2e/other/test_health_lifecycle_e2e.py | 65 +++++++++++++++++ tests/e2e/other/test_master_key_auth_e2e.py | 37 ++++++++++ 6 files changed, 214 insertions(+) create mode 100644 tests/e2e/other/conftest.py create mode 100644 tests/e2e/other/other_client.py create mode 100644 tests/e2e/other/test_health_lifecycle_e2e.py create mode 100644 tests/e2e/other/test_master_key_auth_e2e.py diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 47f3c74d7f1..bf34a896771 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -18,6 +18,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) - `load/` - throughput/performance under concurrency: drives real concurrent traffic through the whole stack with Locust and asserts a throughput SLO; marked `load` so the parent conftest collects it last and it never perturbs latency-sensitive suites +- `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 99757a07e64..321c587eea6 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -975,3 +975,23 @@ class SCIMGroupResponse(BaseModel): model_config = ConfigDict(extra="ignore") id: str displayName: str | None = None + + +# ---------- health / lifecycle ---------- + + +class ReadinessResponse(BaseModel): + """GET /health/readiness (public probe). The low-detail payload a load + balancer sees: `status` plus the resolved DB state (`connected`, + `disconnected`, or `Not connected`).""" + + status: str + db: str | None = None + + +class ReadinessDetailsResponse(ReadinessResponse): + """GET /health/readiness/details (authenticated). Extends the public payload + with the diagnostics only an authenticated caller may read.""" + + litellm_version: str | None = None + success_callbacks: list[str] = [] diff --git a/tests/e2e/other/conftest.py b/tests/e2e/other/conftest.py new file mode 100644 index 00000000000..9141b6e364e --- /dev/null +++ b/tests/e2e/other/conftest.py @@ -0,0 +1,18 @@ +"""`other` suite's `client` fixture. + +Lifecycle (resources/scoped_key), proxy liveness gate, and the e2e/covers +markers all live in the parent tests/e2e/conftest.py. OtherClient holds the +shared ProxyClient so anything these tests create tears down through it. +""" + +from __future__ import annotations + +import pytest + +from other_client import OtherClient, build_client +from proxy_client import ProxyClient + + +@pytest.fixture(scope="session") +def client(proxy: ProxyClient) -> OtherClient: + return build_client(proxy) diff --git a/tests/e2e/other/other_client.py b/tests/e2e/other/other_client.py new file mode 100644 index 00000000000..1aa83ac42c7 --- /dev/null +++ b/tests/e2e/other/other_client.py @@ -0,0 +1,73 @@ +"""Client for the `other` holding-pen suite: the auth gate (master key vs an +invalid key on an admin route) and the process-lifecycle health probes +(liveness, public readiness, authenticated readiness diagnostics). + +Holds the shared ProxyClient so `resources` / `scoped_key` still clean up, and +adds only the routes these behaviors need. The health probes deliberately send +no auth header (public routes), so they go through the transport with an empty +headers model rather than a bearer. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from e2e_http import NoBody, ProbeResult, Result +from models import ( + ReadinessDetailsResponse, + ReadinessResponse, + UserListParams, + UserListResponse, +) +from proxy_client import ProxyClient + + +@dataclass(frozen=True, slots=True) +class OtherClient: + proxy: ProxyClient + + def liveness(self) -> ProbeResult: + """GET /health/liveliness. Unauthenticated; the probe returns status + + raw body so the test can assert the worker reports itself alive.""" + return self.proxy.transport.probe("/health/liveliness", params=NoBody()) + + def readiness_public(self) -> Result[ReadinessResponse]: + """GET /health/readiness with no credential at all, proving the probe is + safe to expose to an unauthenticated load balancer.""" + return self.proxy.transport.get( + "/health/readiness", + headers=NoBody(), + params=NoBody(), + response_type=ReadinessResponse, + ) + + def readiness_details(self, key: str) -> Result[ReadinessDetailsResponse]: + return self.proxy.transport.get( + "/health/readiness/details", + headers=self.proxy.transport.bearer(key), + params=NoBody(), + response_type=ReadinessDetailsResponse, + ) + + def readiness_details_unauthenticated(self) -> Result[ReadinessDetailsResponse]: + return self.proxy.transport.get( + "/health/readiness/details", + headers=NoBody(), + params=NoBody(), + response_type=ReadinessDetailsResponse, + ) + + def list_users_as(self, key: str) -> Result[UserListResponse]: + """GET /user/list under `key`. Admin-only, so it doubles as the master + key's authorization proof: the master key (proxy admin) reads it, a + non-matching key is rejected before it ever reaches the handler.""" + return self.proxy.transport.get( + "/user/list", + headers=self.proxy.transport.bearer(key), + params=UserListParams(user_ids="e2e-test-user"), + response_type=UserListResponse, + ) + + +def build_client(proxy: ProxyClient) -> OtherClient: + return OtherClient(proxy=proxy) diff --git a/tests/e2e/other/test_health_lifecycle_e2e.py b/tests/e2e/other/test_health_lifecycle_e2e.py new file mode 100644 index 00000000000..2551352e8fa --- /dev/null +++ b/tests/e2e/other/test_health_lifecycle_e2e.py @@ -0,0 +1,65 @@ +"""Live e2e: the process-lifecycle probes Kubernetes and load balancers depend on. + +Liveness and public readiness must answer without a credential (a load balancer +has none), and public readiness must distinguish a healthy worker from one whose +DB is unreachable by reporting the resolved DB state. The detailed readiness +route, by contrast, is authenticated: it exposes diagnostics (version, callbacks, +DB) and must reject an anonymous caller. The suite runs against a proxy configured +with a real database, so a healthy readiness payload reports the DB as connected; +a regression that stopped checking the DB, or dropped the public exposure, fails +here. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import MASTER_KEY +from e2e_http import UnauthorizedError, unwrap +from other_client import OtherClient + +pytestmark = pytest.mark.e2e + + +class TestHealthLifecycle: + @pytest.mark.covers("other.lifecycle.liveness.ping") + def test_liveness_reports_alive_without_auth(self, client: OtherClient) -> None: + probe = client.liveness() + assert probe.status_code == 200, ( + f"liveness must answer 200 for an unauthenticated probe, got " + f"{probe.status_code}: {probe.body[:200]}" + ) + assert "alive" in probe.body.lower(), ( + f"liveness body must confirm the worker is alive, got {probe.body[:200]}" + ) + + @pytest.mark.covers("other.lifecycle.readiness.public_probe") + def test_readiness_is_reachable_without_credentials(self, client: OtherClient) -> None: + readiness = unwrap(client.readiness_public()) + assert readiness.status == "healthy", ( + f"public readiness must report a healthy worker, got status {readiness.status!r}" + ) + + @pytest.mark.covers("other.lifecycle.readiness.reports_db_status") + def test_readiness_reports_connected_db(self, client: OtherClient) -> None: + readiness = unwrap(client.readiness_public()) + assert readiness.db == "connected", ( + "readiness must report the configured database as connected so an " + f"orchestrator can tell a healthy worker from a DB-unreachable one, got {readiness.db!r}" + ) + + @pytest.mark.covers("other.lifecycle.readiness_details.authenticated_diagnostics") + def test_readiness_details_require_auth_and_expose_diagnostics(self, client: OtherClient) -> None: + anonymous = client.readiness_details_unauthenticated() + assert isinstance(anonymous, UnauthorizedError), ( + f"/health/readiness/details must reject an unauthenticated caller, got {anonymous}" + ) + + details = unwrap(client.readiness_details(MASTER_KEY)) + assert details.status == "healthy", f"authenticated readiness status must be healthy, got {details.status!r}" + assert details.litellm_version is not None, ( + "authenticated diagnostics must expose the litellm version" + ) + assert details.db == "connected", ( + f"authenticated diagnostics must report the DB as connected, got {details.db!r}" + ) diff --git a/tests/e2e/other/test_master_key_auth_e2e.py b/tests/e2e/other/test_master_key_auth_e2e.py new file mode 100644 index 00000000000..6ab33c9b62a --- /dev/null +++ b/tests/e2e/other/test_master_key_auth_e2e.py @@ -0,0 +1,37 @@ +"""Live e2e: the master key authenticates and is treated as a proxy admin, and a +key that is not the master key is rejected before reaching the handler. + +/user/list is admin-only, so it proves both halves of the master-key contract in +one route: the master key reads it (authenticated + authorized as admin), while a +freshly minted, never-provisioned token is denied 401 by the auth layer. The +invalid case uses a unique, master-key-shaped token so the check exercises the +credential comparison rather than a value that could collide with a real key. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import MASTER_KEY, unique_marker +from e2e_http import UnauthorizedError, unwrap +from other_client import OtherClient + +pytestmark = pytest.mark.e2e + + +class TestMasterKeyAuth: + @pytest.mark.covers("other.auth.master_key.valid_allows") + def test_master_key_authenticates_and_grants_admin_route(self, client: OtherClient) -> None: + listing = unwrap(client.list_users_as(MASTER_KEY)) + assert listing.total >= 0, ( + "master key reached the admin /user/list handler but the response did not " + f"carry a user count: {listing}" + ) + + @pytest.mark.covers("other.auth.master_key.invalid_denied") + def test_non_matching_master_key_is_denied(self, client: OtherClient) -> None: + bogus = f"sk-{unique_marker()}" + result = client.list_users_as(bogus) + assert isinstance(result, UnauthorizedError), ( + f"a token that is not the master key must be rejected with 401, got {result}" + )